From 5d7fdf9350f544e33e5e64e73c217d8f33936df6 Mon Sep 17 00:00:00 2001 From: chickeyton Date: Sat, 5 Sep 2026 16:20:21 +0800 Subject: [PATCH] [Core] Graduate MiniCPM-o 4.5 and PersonaPlex full-duplex serving out of experimental (#6196) Signed-off-by: chickeyton Signed-off-by: Gao Han Co-authored-by: Claude Fable 5 Co-authored-by: Gao Han --- .buildkite/cuda/test-merge.yml | 15 + .buildkite/cuda/test-ready.yml | 19 +- README.md | 2 +- docs/.nav.yml | 1 + docs/README.md | 3 +- docs/api/README.md | 30 + docs/cli/bench/serve.md | 32 +- docs/design/architecture_overview.md | 2 +- .../design/fullduplex-personaplex.md | 44 +- .../DESIGN.md => docs/design/fullduplex.md | 120 +- docs/design/index.md | 2 + docs/design/module/engine_orchestration.md | 6 +- docs/design/module/entrypoints.md | 9 + docs/mkdocs/hooks/generate_api_readme.py | 7 + docs/serving/README.md | 13 +- docs/serving/full_duplex_api.md | 18 +- docs/serving/realtime_api.md | 4 +- docs/serving/realtime_duplex_api.md | 1217 ++++++++++++ docs/serving/standalone_servers.md | 62 +- docs/usage/faq.md | 2 +- .../personaplex/personaplex_offline.py | 77 - examples/online_serving/barge_in_client.py | 272 +++ .../online_serving/barge_in_client_flow.md | 54 + examples/online_serving/minicpmo/README.md | 8 +- .../online_serving/minicpmo/barge_in_serve.sh | 25 + .../minicpmo/realtime_duplex_demo.py | 131 +- .../minicpmo/realtime_web/app/static/app.js | 4 +- examples/online_serving/personaplex/README.md | 83 +- .../personaplex/duplex_client.py | 114 -- .../online_serving/personaplex/run_server.sh | 15 - recipes/NVIDIA/PersonaPlex.md | 112 +- tests/benchmarks/patch/test_patch.py | 6 +- tests/benchmarks/test_omniinteract.py | 118 +- tests/clients/__init__.py | 2 + tests/clients/test_duplex_client.py | 895 +++++++++ tests/clients/test_model_session_configs.py | 70 + .../personaplex/test_import_boundary.py | 30 - tests/e2e/features/fullduplex/test_client.py | 14 +- .../fullduplex/test_core_contracts.py | 4 +- .../fullduplex/test_personaplex_adapter.py | 262 --- .../fullduplex/test_personaplex_batched.py | 178 -- .../fullduplex/test_personaplex_elastic.py | 245 --- .../test_personaplex_server_lease.py | 101 - .../fullduplex/test_personaplex_session.py | 97 - .../helpers/minicpmo_4_5_duplex.py | 29 +- .../minicpmo_realtime_duplex_scenarios.py | 54 +- .../personaplex_realtime_duplex.py | 79 +- ..._minicpmo_realtime_duplex_multi_session.py | 8 +- .../online_serving/test_duplex_client_live.py | 139 ++ .../test_minicpmo_4_5_duplex.py | 12 +- .../test_minicpmo_realtime_duplex_drivers.py | 4 +- .../minicpmo45 => engine/duplex}/__init__.py | 0 .../duplex}/test_duplex_control_client.py | 8 +- .../duplex}/test_duplex_control_plane.py | 18 +- .../duplex}/test_duplex_deploy_config.py | 2 +- .../duplex}/test_duplex_intermediate.py | 5 +- .../duplex}/test_duplex_lease.py | 10 +- .../duplex}/test_duplex_runtime.py | 13 +- .../engine/test_async_omni_engine_outputs.py | 23 +- tests/engine/test_duplex_import_boundary.py | 121 +- tests/engine/test_orchestrator.py | 30 +- .../test_orchestrator_segment_stage_keying.py | 6 +- .../test_orchestrator_stage_input_bridge.py | 12 +- .../duplex}/test_duplex_audio.py | 4 +- .../duplex}/test_runtime_adapter_boundary.py | 27 +- .../duplex}/test_runtime_control_redaction.py | 6 +- .../duplex}/test_websocket_actor.py | 4 +- .../openai/test_duplex_protocol.py | 92 +- .../openai/test_duplex_session_attachment.py | 6 +- .../openai_api/test_duplex_capability.py | 41 +- .../openai_api/test_duplex_handler.py | 196 +- tests/entrypoints/test_async_omni_duplex.py | 19 +- .../test_duplex_fence_propagation.py | 9 +- ...st_minicpmo_realtime_duplex_simple_demo.py | 19 +- .../test_minicpmo_realtime_web_static.py | 5 +- .../models/minicpmo_4_5/duplex/__init__.py | 0 .../duplex}/test_commit_policy.py | 5 +- .../models/minicpmo_4_5/duplex}/test_input.py | 4 +- .../models/minicpmo_4_5/duplex/test_policy.py | 22 + .../models/minicpmo_4_5/test_pipeline.py | 2 +- .../models/nemotron_voicechat/__init__.py | 2 + .../nemotron_voicechat/duplex/__init__.py | 2 + .../duplex}/test_data_plane.py | 9 +- .../nemotron_voicechat/duplex}/test_input.py | 5 +- .../duplex}/test_runtime_contract.py | 8 +- .../models/personaplex/__init__.py | 2 + .../models/personaplex/duplex/__init__.py | 2 + .../duplex}/test_stage0_runtime.py | 6 +- .../duplex}/test_unified_runtime.py | 29 +- .../personaplex/test_streaming_code2wav.py | 2 +- tests/worker/test_native_duplex_hooks.py | 64 +- tools/pre_commit/check_forbidden_imports.py | 26 +- vllm_omni/benchmarks/omniinteract.py | 272 ++- vllm_omni/benchmarks/patch/patch.py | 112 +- vllm_omni/clients/__init__.py | 9 + vllm_omni/clients/duplex.py | 1688 +++++++++++++++++ vllm_omni/clients/minicpmo_4_5.py | 24 + vllm_omni/clients/personaplex.py | 22 + vllm_omni/engine/async_omni_engine.py | 21 +- vllm_omni/engine/duplex/__init__.py | 4 + .../engine => engine/duplex}/contracts.py | 4 +- .../duplex/control_client.py} | 8 +- .../duplex/control_plane.py} | 22 +- .../engine => engine/duplex}/intermediate.py | 3 + .../engine => engine/duplex}/lease.py | 4 +- .../engine => engine/duplex}/messages.py | 2 +- .../duplex/runtime.py} | 18 +- .../duplex/session.py} | 8 +- vllm_omni/engine/orchestrator.py | 52 +- vllm_omni/entrypoints/async_omni.py | 29 +- vllm_omni/entrypoints/cli/benchmark/serve.py | 7 +- .../openai => entrypoints/duplex}/__init__.py | 2 +- .../openai => entrypoints/duplex}/audio.py | 2 +- .../capability.py} | 2 +- .../duplex}/chat_fallback.py | 3 +- .../duplex}/commit_policy.py | 3 + .../openai => entrypoints/duplex}/protocol.py | 64 +- .../duplex}/realtime_input.py | 11 +- .../duplex}/realtime_output.py | 38 +- .../duplex}/realtime_session.py | 9 +- .../duplex}/realtime_state.py | 13 +- .../duplex}/runtime_adapter.py | 3 + .../duplex}/runtime_bridge.py | 38 +- .../openai => entrypoints/duplex}/serving.py | 60 +- .../duplex}/session_attachment.py | 2 +- .../duplex}/session_runner.py | 26 +- .../openai => entrypoints/duplex}/vad.py | 3 + .../duplex}/websocket.py | 2 +- .../duplex_request_client.py} | 14 +- vllm_omni/entrypoints/openai/api_server.py | 5 +- vllm_omni/experimental/fullduplex/README.md | 65 +- vllm_omni/experimental/fullduplex/client.py | 6 +- .../fullduplex/engine/__init__.py | 4 - .../fullduplex/personaplex/__init__.py | 40 - .../fullduplex/personaplex/adapter.py | 177 -- .../fullduplex/personaplex/engine.py | 74 - .../fullduplex/personaplex/runtime.py | 473 ----- .../personaplex/serving/__init__.py | 15 - .../fullduplex/personaplex/serving/batched.py | 184 -- .../fullduplex/personaplex/serving/server.py | 539 ------ .../fullduplex/personaplex/session.py | 124 -- .../duplex_sampling.py} | 2 +- .../models/minicpmo_4_5/duplex}/__init__.py | 2 +- .../models/minicpmo_4_5/duplex}/adapter.py | 8 +- .../minicpmo_4_5/duplex/capabilities.py | 48 + .../models/minicpmo_4_5/duplex}/compat.py | 3 + .../models/minicpmo_4_5/duplex}/data_plane.py | 7 +- .../models/minicpmo_4_5/duplex}/input.py | 3 + .../models/minicpmo_4_5/duplex}/policy.py | 0 .../models/minicpmo_4_5/duplex}/runtime.py | 4 +- .../minicpmo_4_5/duplex}/serving_adapter.py | 16 +- .../models/minicpmo_4_5/duplex}/session.py | 5 +- .../models/minicpmo_4_5/duplex}/stage0.py | 4 +- .../models/minicpmo_4_5/minicpmo_4_5_omni.py | 12 +- .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 4 +- .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 2 +- .../models/minicpmo_4_5/pipeline.py | 6 +- .../nemotron_voicechat/duplex}/__init__.py | 2 +- .../nemotron_voicechat/duplex}/data_plane.py | 6 +- .../nemotron_voicechat/duplex}/input.py | 2 +- .../nemotron_voicechat/duplex}/runtime.py | 8 +- .../duplex}/serving_adapter.py | 10 +- .../nemotron_voicechat_thinker.py | 4 +- .../models/nemotron_voicechat/pipeline.py | 6 +- .../models/personaplex/duplex/__init__.py | 41 + .../models/personaplex/duplex}/config.py | 2 +- .../models/personaplex/duplex}/data_plane.py | 4 +- .../models/personaplex/duplex}/input.py | 2 +- .../models/personaplex/duplex}/policy.py | 2 +- .../personaplex/duplex}/runtime_extension.py | 6 +- .../personaplex/duplex}/serving_adapter.py | 17 +- .../models/personaplex/duplex}/stage0.py | 4 +- .../personaplex/personaplex_code2wav.py | 4 +- .../models/personaplex/personaplex_talker.py | 4 +- .../models/personaplex/pipeline.py | 6 +- .../minicpmo_4_5_omni.py | 4 +- .../output.py => outputs/duplex.py} | 4 +- .../npu/worker/npu_ar_model_runner.py | 2 +- vllm_omni/worker/gpu_ar_model_runner.py | 2 +- 179 files changed, 6462 insertions(+), 3832 deletions(-) rename vllm_omni/experimental/fullduplex/personaplex/DESIGN.md => docs/design/fullduplex-personaplex.md (83%) rename vllm_omni/experimental/fullduplex/DESIGN.md => docs/design/fullduplex.md (91%) create mode 100644 docs/serving/realtime_duplex_api.md delete mode 100644 examples/offline_inference/personaplex/personaplex_offline.py create mode 100644 examples/online_serving/barge_in_client.py create mode 100644 examples/online_serving/barge_in_client_flow.md create mode 100644 examples/online_serving/minicpmo/barge_in_serve.sh delete mode 100644 examples/online_serving/personaplex/duplex_client.py delete mode 100755 examples/online_serving/personaplex/run_server.sh create mode 100644 tests/clients/__init__.py create mode 100644 tests/clients/test_duplex_client.py create mode 100644 tests/clients/test_model_session_configs.py delete mode 100644 tests/e2e/features/fullduplex/personaplex/test_import_boundary.py delete mode 100644 tests/e2e/features/fullduplex/test_personaplex_adapter.py delete mode 100644 tests/e2e/features/fullduplex/test_personaplex_batched.py delete mode 100644 tests/e2e/features/fullduplex/test_personaplex_elastic.py delete mode 100644 tests/e2e/features/fullduplex/test_personaplex_server_lease.py delete mode 100644 tests/e2e/features/fullduplex/test_personaplex_session.py create mode 100644 tests/e2e/online_serving/test_duplex_client_live.py rename tests/{e2e/features/fullduplex/minicpmo45 => engine/duplex}/__init__.py (100%) rename tests/{e2e/features/fullduplex/engine => engine/duplex}/test_duplex_control_client.py (92%) rename tests/{e2e/features/fullduplex/engine => engine/duplex}/test_duplex_control_plane.py (99%) rename tests/{e2e/features/fullduplex/engine => engine/duplex}/test_duplex_deploy_config.py (97%) rename tests/{e2e/features/fullduplex/engine => engine/duplex}/test_duplex_intermediate.py (69%) rename tests/{e2e/features/fullduplex/engine => engine/duplex}/test_duplex_lease.py (95%) rename tests/{e2e/features/fullduplex/engine => engine/duplex}/test_duplex_runtime.py (97%) rename tests/{e2e/features/fullduplex/openai => entrypoints/duplex}/test_duplex_audio.py (91%) rename tests/{e2e/features/fullduplex/openai => entrypoints/duplex}/test_runtime_adapter_boundary.py (79%) rename tests/{e2e/features/fullduplex/openai => entrypoints/duplex}/test_runtime_control_redaction.py (93%) rename tests/{e2e/features/fullduplex/openai => entrypoints/duplex}/test_websocket_actor.py (97%) create mode 100644 tests/model_executor/models/minicpmo_4_5/duplex/__init__.py rename tests/{e2e/features/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/test_commit_policy.py (94%) rename tests/{e2e/features/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/test_input.py (97%) create mode 100644 tests/model_executor/models/minicpmo_4_5/duplex/test_policy.py create mode 100644 tests/model_executor/models/nemotron_voicechat/__init__.py create mode 100644 tests/model_executor/models/nemotron_voicechat/duplex/__init__.py rename tests/{e2e/features/fullduplex/nemotron_voicechat => model_executor/models/nemotron_voicechat/duplex}/test_data_plane.py (91%) rename tests/{e2e/features/fullduplex/nemotron_voicechat => model_executor/models/nemotron_voicechat/duplex}/test_input.py (92%) rename tests/{e2e/features/fullduplex/nemotron_voicechat => model_executor/models/nemotron_voicechat/duplex}/test_runtime_contract.py (92%) create mode 100644 tests/model_executor/models/personaplex/__init__.py create mode 100644 tests/model_executor/models/personaplex/duplex/__init__.py rename tests/{e2e/features/fullduplex/personaplex => model_executor/models/personaplex/duplex}/test_stage0_runtime.py (97%) rename tests/{e2e/features/fullduplex/personaplex => model_executor/models/personaplex/duplex}/test_unified_runtime.py (91%) rename tests/{e2e/features/fullduplex => model_executor/models}/personaplex/test_streaming_code2wav.py (99%) create mode 100644 vllm_omni/clients/__init__.py create mode 100644 vllm_omni/clients/duplex.py create mode 100644 vllm_omni/clients/minicpmo_4_5.py create mode 100644 vllm_omni/clients/personaplex.py create mode 100644 vllm_omni/engine/duplex/__init__.py rename vllm_omni/{experimental/fullduplex/engine => engine/duplex}/contracts.py (98%) rename vllm_omni/{experimental/fullduplex/engine/duplex_control_client.py => engine/duplex/control_client.py} (96%) rename vllm_omni/{experimental/fullduplex/engine/duplex_control_plane.py => engine/duplex/control_plane.py} (99%) rename vllm_omni/{experimental/fullduplex/engine => engine/duplex}/intermediate.py (96%) rename vllm_omni/{experimental/fullduplex/engine => engine/duplex}/lease.py (96%) rename vllm_omni/{experimental/fullduplex/engine => engine/duplex}/messages.py (97%) rename vllm_omni/{experimental/fullduplex/engine/duplex_runtime.py => engine/duplex/runtime.py} (91%) rename vllm_omni/{experimental/fullduplex/engine/duplex_session.py => engine/duplex/session.py} (98%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/__init__.py (61%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/audio.py (99%) rename vllm_omni/entrypoints/{openai/duplex_capability.py => duplex/capability.py} (93%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/chat_fallback.py (98%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/commit_policy.py (91%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/protocol.py (97%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/realtime_input.py (99%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/realtime_output.py (97%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/realtime_session.py (95%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/realtime_state.py (94%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/runtime_adapter.py (98%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/runtime_bridge.py (97%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/serving.py (97%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/session_attachment.py (99%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/session_runner.py (99%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/vad.py (98%) rename vllm_omni/{experimental/fullduplex/openai => entrypoints/duplex}/websocket.py (99%) rename vllm_omni/{experimental/fullduplex/request_client.py => entrypoints/duplex_request_client.py} (97%) delete mode 100644 vllm_omni/experimental/fullduplex/engine/__init__.py delete mode 100644 vllm_omni/experimental/fullduplex/personaplex/__init__.py delete mode 100644 vllm_omni/experimental/fullduplex/personaplex/adapter.py delete mode 100644 vllm_omni/experimental/fullduplex/personaplex/engine.py delete mode 100644 vllm_omni/experimental/fullduplex/personaplex/runtime.py delete mode 100644 vllm_omni/experimental/fullduplex/personaplex/serving/__init__.py delete mode 100644 vllm_omni/experimental/fullduplex/personaplex/serving/batched.py delete mode 100644 vllm_omni/experimental/fullduplex/personaplex/serving/server.py delete mode 100644 vllm_omni/experimental/fullduplex/personaplex/session.py rename vllm_omni/{experimental/fullduplex/model_executor.py => model_executor/duplex_sampling.py} (98%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/__init__.py (86%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/adapter.py (97%) create mode 100644 vllm_omni/model_executor/models/minicpmo_4_5/duplex/capabilities.py rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/compat.py (92%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/data_plane.py (99%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/input.py (99%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/policy.py (100%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/runtime.py (98%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/serving_adapter.py (83%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/session.py (91%) rename vllm_omni/{experimental/fullduplex/minicpmo45 => model_executor/models/minicpmo_4_5/duplex}/stage0.py (99%) rename vllm_omni/{experimental/fullduplex/nemotron_voicechat => model_executor/models/nemotron_voicechat/duplex}/__init__.py (60%) rename vllm_omni/{experimental/fullduplex/nemotron_voicechat => model_executor/models/nemotron_voicechat/duplex}/data_plane.py (98%) rename vllm_omni/{experimental/fullduplex/nemotron_voicechat => model_executor/models/nemotron_voicechat/duplex}/input.py (99%) rename vllm_omni/{experimental/fullduplex/nemotron_voicechat => model_executor/models/nemotron_voicechat/duplex}/runtime.py (95%) rename vllm_omni/{experimental/fullduplex/nemotron_voicechat => model_executor/models/nemotron_voicechat/duplex}/serving_adapter.py (97%) create mode 100644 vllm_omni/model_executor/models/personaplex/duplex/__init__.py rename vllm_omni/{experimental/fullduplex/personaplex => model_executor/models/personaplex/duplex}/config.py (97%) rename vllm_omni/{experimental/fullduplex/personaplex => model_executor/models/personaplex/duplex}/data_plane.py (98%) rename vllm_omni/{experimental/fullduplex/personaplex => model_executor/models/personaplex/duplex}/input.py (98%) rename vllm_omni/{experimental/fullduplex/personaplex => model_executor/models/personaplex/duplex}/policy.py (96%) rename vllm_omni/{experimental/fullduplex/personaplex => model_executor/models/personaplex/duplex}/runtime_extension.py (95%) rename vllm_omni/{experimental/fullduplex/personaplex => model_executor/models/personaplex/duplex}/serving_adapter.py (93%) rename vllm_omni/{experimental/fullduplex/personaplex => model_executor/models/personaplex/duplex}/stage0.py (99%) rename vllm_omni/{experimental/fullduplex/output.py => outputs/duplex.py} (84%) diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 8d002f31c..d2a3ffb02 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -183,6 +183,21 @@ steps: - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py - vllm_omni/deploy/minicpmo_4_5.yaml - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py + - vllm_omni/engine/duplex/ + - vllm_omni/entrypoints/duplex/ + # Public duplex client library + its unit suites: client-side + # changes must re-run the duplex e2e group. The live client probe + # (test_duplex_client_live.py, single GPU, core tier) runs in the + # ready pipeline's duplex group. + - vllm_omni/clients/ + - tests/clients/ + - vllm_omni/entrypoints/duplex_request_client.py + - vllm_omni/outputs/duplex.py + - vllm_omni/model_executor/duplex_sampling.py + - tests/engine/duplex/ + - tests/entrypoints/duplex/ + - tests/model_executor/models/minicpmo_4_5/duplex/ + # Legacy Realtime probe client + video stacking used by the e2e drivers. - vllm_omni/experimental/fullduplex/ - vllm_omni/model_executor/models/minicpmo_4_5/ - vllm_omni/model_executor/models/step_audio2/ diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index f9e64159c..1ea89ae97 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -214,6 +214,21 @@ steps: - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py - vllm_omni/deploy/minicpmo_4_5.yaml - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py + - vllm_omni/engine/duplex/ + - vllm_omni/entrypoints/duplex/ + # Public duplex client library + its unit suites: client-side + # changes must re-run the duplex e2e group, which includes the + # live client probe (test_duplex_client_live.py, single GPU). + - vllm_omni/clients/ + - tests/clients/ + - tests/e2e/online_serving/test_duplex_client_live.py + - vllm_omni/entrypoints/duplex_request_client.py + - vllm_omni/outputs/duplex.py + - vllm_omni/model_executor/duplex_sampling.py + - tests/engine/duplex/ + - tests/entrypoints/duplex/ + - tests/model_executor/models/minicpmo_4_5/duplex/ + # Legacy Realtime probe client + video stacking used by the e2e drivers. - vllm_omni/experimental/fullduplex/ - vllm_omni/model_executor/models/minicpmo_4_5/ - vllm_omni/model_executor/models/step_audio2/ @@ -231,8 +246,8 @@ steps: - vllm_omni/config/stage_config.py commands: - | - timeout 20m bash -c " - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model' + timeout 35m bash -c " + pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_duplex_client_live.py -m 'core_model and cuda' --run-level 'core_model' " mirror_hardwares: h100_1 diff --git a/README.md b/README.md index 7a21c8ab6..5063f9a3f 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ vLLM-Omni is flexible and easy to use with: - Tensor, pipeline, data and expert parallelism support for distributed inference - Streaming outputs - OpenAI-compatible API server -- Full-duplex realtime serving with streaming audio input and output (experimental) +- [Full-duplex realtime serving](docs/serving/realtime_duplex_api.md) with streaming audio input and output vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including: diff --git a/docs/.nav.yml b/docs/.nav.yml index 94c4de213..b5a83c338 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -17,6 +17,7 @@ nav: - Streaming and Realtime: - Realtime Audio: serving/realtime_api.md - Full Duplex: serving/full_duplex_api.md + - Realtime Duplex Reference: serving/realtime_duplex_api.md - Streaming Video Input: serving/video_stream_api.md - Streaming Video Output: serving/streaming_video_output_api.md - OpenPI Robot Policy: serving/openpi_api.md diff --git a/docs/README.md b/docs/README.md index 7eeb0bd70..90a8b6304 100644 --- a/docs/README.md +++ b/docs/README.md @@ -23,7 +23,6 @@ Easy, fast, and cheap omni-modality model serving for everyone Fork

- ## About [vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni is a framework that extends its support for omni-modality model inference and serving: @@ -52,7 +51,7 @@ vLLM-Omni is flexible and easy to use with: - Tensor, pipeline, data and expert parallelism support for distributed inference - Streaming outputs - OpenAI-compatible API server -- Full-duplex realtime serving with streaming audio input and output (experimental) +- [Full-duplex realtime serving](serving/realtime_duplex_api.md) with streaming audio input and output vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including: diff --git a/docs/api/README.md b/docs/api/README.md index 4a34e3f31..b2d4ea1ba 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -15,6 +15,36 @@ Main entry points for vLLM-Omni inference and serving. - [vllm_omni.entrypoints.omni_base.OmniBase][] - [vllm_omni.entrypoints.pd_utils.PDDisaggregationMixin][] +## Clients + +Client libraries that talk to a running vLLM-Omni server (the full-duplex Realtime client and its per-model session presets). + +- [vllm_omni.clients.duplex.AudioDelta][] +- [vllm_omni.clients.duplex.AudioFormat][] +- [vllm_omni.clients.duplex.ConnectionResumed][] +- [vllm_omni.clients.duplex.DuplexClient][] +- [vllm_omni.clients.duplex.DuplexClientError][] +- [vllm_omni.clients.duplex.DuplexConnectionError][] +- [vllm_omni.clients.duplex.DuplexEvent][] +- [vllm_omni.clients.duplex.DuplexProtocolError][] +- [vllm_omni.clients.duplex.DuplexSessionClosedError][] +- [vllm_omni.clients.duplex.ErrorEvent][] +- [vllm_omni.clients.duplex.EventCollector][] +- [vllm_omni.clients.duplex.ListenDecision][] +- [vllm_omni.clients.duplex.ReconnectPolicy][] +- [vllm_omni.clients.duplex.ResponseCreated][] +- [vllm_omni.clients.duplex.ResponseDone][] +- [vllm_omni.clients.duplex.ResponseHandle][] +- [vllm_omni.clients.duplex.SessionClosed][] +- [vllm_omni.clients.duplex.SessionConfig][] +- [vllm_omni.clients.duplex.SessionCreated][] +- [vllm_omni.clients.duplex.SessionExpired][] +- [vllm_omni.clients.duplex.SessionResumed][] +- [vllm_omni.clients.duplex.SpeakDecision][] +- [vllm_omni.clients.duplex.TextDelta][] +- [vllm_omni.clients.duplex.TranscriptDelta][] +- [vllm_omni.clients.duplex.WebSocketTransport][] + ## Inputs Input data structures for multi-modal inputs. diff --git a/docs/cli/bench/serve.md b/docs/cli/bench/serve.md index efb23df00..b9c590095 100644 --- a/docs/cli/bench/serve.md +++ b/docs/cli/bench/serve.md @@ -1,11 +1,15 @@ # vLLM-Omni Benchmark CLI Guide + The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models. ## Notes + vLLM-Omni registers the `openai-chat-omni`, `openai-audio-speech`, `openai-image-edits-omni`, `daily-omni`, and `openai-realtime-duplex` serving benchmark backends. It also adds the `omniinteract` dataset. ## Basic Parameter Description + You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below: + - `--omni` Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio. @@ -77,8 +81,8 @@ Specify to save benchmark results to a json file - `--random-prefix-len` Number of fixed prefix tokens before the random context in a request. The total input length is the sum of random-prefix-len and a random - context length sampled from [input_len * (1 - range_ratio), - input_len * (1 + range_ratio)].Only the random and random-mm modes + context length sampled from [input_len *(1 - range_ratio), + input_len* (1 + range_ratio)].Only the random and random-mm modes support this parameter. - `--random-input-len` @@ -91,7 +95,7 @@ Specify to save benchmark results to a json file Range ratio for sampling input/output length, used only for random sampling. Must be in the range [0, 1) to define a symmetric sampling range - [length * (1 - range_ratio), length * (1 + range_ratio)]. + [length *(1 - range_ratio), length* (1 + range_ratio)]. Only the random and random-mm modes support this parameter. - `--random-mm-base-items-per-request` @@ -137,6 +141,7 @@ Specify to save benchmark results to a json file ## Usage Examples ### Online Benchmark +
Show more @@ -162,7 +167,9 @@ vllm bench serve \ --dataset-path ShareGPT_V3_unfiltered_cleaned_split.json \ --percentile-metrics ttft,tpot,itl,e2el ``` + If successful, you will see the following output: + ```text ============ Serving Benchmark Result ============ Successful requests: 2 @@ -264,6 +271,7 @@ Median AUDIO_RTF: 0.47 P99 AUDIO_RTF: 0.48 ================================================== ``` + Notes: We use audio generation time / audio duration to calculate RTF. @@ -301,8 +309,12 @@ options apply. Use `--omniinteract-require-response` only for functional E2E cas Each completed case writes `output.wav`, `wav_transcript.json`, `events.json`, `result.json`, and a final `.done` marker under `--omniinteract-output-dir`. The root also contains `batch_summary.json` and `official_eval_manifest.jsonl`; failed cases write `.failed.json`. Runs sharing one output root are serialized. Completion validates transport, response lifecycle, and artifacts, -not answer accuracy. Transcript timestamps are serialized playback-queue times. Clipped or cancelled outputs are ineligible and -omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video horizon. +not answer accuracy. Transcript timestamps are serialized playback-queue times. Playback ACKs report cumulative progress +incrementally along that serialized clock, like a live listener; the first ack for a response goes out as soon as its audio +arrives, checkpointing the response's history position so a later committed user input updates it in place. A residual +`playback_ack_too_late` rejection is recorded as an artifact warning rather than failing the case. Clipped or cancelled +outputs are ineligible and omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video +horizon. TTFT, TTFP, and RTF start at client receipt of `response.created`. TPOT/ITL use engine stage-0 timing; ITL is emitted only when every token interval is present. @@ -339,8 +351,8 @@ Generate synthetic image、video、audio inputs alongside random text prompts to Notes: - Works only with online benchmark via: - - the OpenAI chat backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`. - - the OpenAI edit image backend (`--backend openai-image-edits-omni`) and endpoint `/v1/images/edits`. + - the OpenAI chat backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`. + - the OpenAI edit image backend (`--backend openai-image-edits-omni`) and endpoint `/v1/images/edits`. Start the server (example): @@ -351,6 +363,7 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni It is recommended to use the flag `--ignore-eos` to simulate real responses. You can set the size of the output via the arg `random-output-len`. Then run the benchmarking script: + ```bash vllm bench serve \ --omni \ @@ -420,6 +433,7 @@ How sampling works: - If a modality (e.g., image) reaches its limit from `--random-mm-limit-mm-per-prompt`, all buckets of that modality are excluded and the remaining bucket probabilities are renormalized before continuing. This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`. - The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`. +
### Multi-Stage Benchmark @@ -434,6 +448,7 @@ vllm serve --omni --port 29999 --model ~/Qwen3-Omni-30B-A3B-Instruct ``` Then run the benchmarking script (remember to add the flag `--print-stage`): + ```bash vllm bench serve --omni \ --dataset-name random \ @@ -453,6 +468,7 @@ vllm bench serve --omni \ ``` Besides the "Serving Benchmark Result", there will be "Stage Benchmark Result" below: + ```text ============= Stage Benchmark Result ============= =============== Stage 0 (thinker) ================ @@ -509,7 +525,9 @@ Mean AUDIO_RTF: 0.24 Median AUDIO_RTF: 0.26 P99 AUDIO_RTF: 0.27 ``` + Explanation: + - stage_gen_time: Time from submitting a request to a specific stage to that stage finishing generation. - Serving TTFC (Time to First Chunk): Time from the HTTP request being accepted by the serving frontend to diff --git a/docs/design/architecture_overview.md b/docs/design/architecture_overview.md index d7938b62c..5b14dd8ec 100644 --- a/docs/design/architecture_overview.md +++ b/docs/design/architecture_overview.md @@ -418,7 +418,7 @@ The public interfaces map onto the same engine and stage boundaries: flowchart LR offline["Offline Python
Omni.generate()"] --> engine["AsyncOmniEngine"] online["OpenAI-compatible serving
vllm serve ... --omni"] --> engine - duplex["Experimental duplex WebSocket
/v1/duplex or realtime duplex"] --> engine + duplex["Duplex WebSocket
/v1/realtime?duplex=1 or /v1/duplex"] --> engine engine --> stages["Configured AR and diffusion stages"] stages --> result["Streaming or final multimodal output"] ``` diff --git a/vllm_omni/experimental/fullduplex/personaplex/DESIGN.md b/docs/design/fullduplex-personaplex.md similarity index 83% rename from vllm_omni/experimental/fullduplex/personaplex/DESIGN.md rename to docs/design/fullduplex-personaplex.md index 623f08353..249554bbb 100644 --- a/vllm_omni/experimental/fullduplex/personaplex/DESIGN.md +++ b/docs/design/fullduplex-personaplex.md @@ -23,16 +23,18 @@ The target is the engine-native path: -> response.audio.delta + response.audio_transcript.delta ``` -The standalone `/api/chat` and `/v1/audio/duplex` server remains a compatibility -surface. It is not evidence that the unified engine path works. +The standalone `/api/chat` and `/v1/audio/duplex` server that accompanied the +original PR was demo-only and has been removed from the tree; the unified +engine path above is the only serving surface. It was never evidence that the +unified engine path works. -## Why configuration-only enablement is invalid +## Why configuration-only enablement was invalid -The current staged pipeline is explicitly turn based. Its Talker reads -`pplex_user_codes`, `pplex_prefill_text`, and `pplex_silence_codes`, but no -production staged input path writes those fields. The voice prompt, persona -prefill, and streaming Mimi state live only in the standalone -`PersonaPlexEngine`. +The staged pipeline as shipped by the original PR was explicitly turn based. +Its Talker read `pplex_user_codes`, `pplex_prefill_text`, and +`pplex_silence_codes`, but no production staged input path wrote those fields. +The voice prompt, persona prefill, and streaming Mimi state lived only in the +standalone `PersonaPlexEngine` (demo-only, since removed). Setting only the following fields would therefore advertise an endpoint whose model never receives the live microphone stream: @@ -56,7 +58,12 @@ The unified implementation supports: - continuous user input while assistant audio is generated or played; - bundled `.pt` voice prompts and a session persona; - greedy text and depformer sampling, matching the current PersonaPlex port; -- `/v1/realtime?duplex=1`; +- `/v1/realtime?duplex=1` (the wire vocabulary is catalogued in the + [Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide); +- the public client preset + `vllm_omni.clients.personaplex.create_duplex_session_config()` (24 kHz + `pcm_f32le` input format, voice prompt, persona) as the canonical + session-config source for `DuplexClient` consumers; - engine lease close, disconnect cleanup, reconnect after cleanup, and explicit response cancellation without cross-session state reuse. @@ -189,10 +196,17 @@ model-neutral native results: ``` It owns per-request audio and text cursors so a cumulative output cannot replay -old audio. PersonaPlex keeps one visible response open while continuous output -arrives. Session close or cancellation terminates that response through the -generic Realtime lifecycle; a codec segment finishing is not a conversational -turn boundary. +old audio. The generic projector turns each native result into the Realtime +`response.audio.delta` / `response.audio_transcript.delta` pair (and +`response.output_text.delta` for text) under one `response_id`; the full +mapping is the name map in the +[Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide. +PersonaPlex keeps one visible response open while continuous output arrives. +Session close or cancellation terminates that response through the generic +Realtime lifecycle; a codec segment finishing is not a conversational turn +boundary. PersonaPlex advertises `supports_barge_in=false` and +`supports_session_resume=false`, so `barge_in`, `turn_detection.server_vad`, +and `session.resume` are rejected on this model. ## Error and lifecycle contracts @@ -210,8 +224,8 @@ turn boundary. - New sessions cannot observe the previous voice, persona, PCM tail, Mimi convolution state, or Talker delayed code frame. -The legacy `PersonaPlexDuplexRuntime.run()` and standalone server drain path -must follow the same exception-safe rule while they remain in the tree. +(The legacy `PersonaPlexDuplexRuntime.run()` and standalone server drain path, +which followed the same exception-safe rule, have been removed from the tree.) ## Testing and acceptance diff --git a/vllm_omni/experimental/fullduplex/DESIGN.md b/docs/design/fullduplex.md similarity index 91% rename from vllm_omni/experimental/fullduplex/DESIGN.md rename to docs/design/fullduplex.md index 916a2a096..2311ee1cf 100644 --- a/vllm_omni/experimental/fullduplex/DESIGN.md +++ b/docs/design/fullduplex.md @@ -1,5 +1,13 @@ # MiniCPM-o 4.5 Full-Duplex Runtime Architecture +> Relocation note: after the H20 validation recorded below, the full-duplex +> implementation moved out of `vllm_omni/experimental/fullduplex/` into the +> stable tree (`vllm_omni/engine/duplex/`, `vllm_omni/entrypoints/duplex/`, +> `vllm_omni/model_executor/models/minicpmo_4_5/duplex/`, and companion +> modules). Module paths in this document reflect the new layout; the runtime +> evidence was produced on the pre-move tree, which is file-identical apart +> from module paths. + ## Purpose This document records the runtime architecture implemented by PR #3907, the @@ -31,8 +39,8 @@ session was rejected and capacity returned after one session closed. The two accepted sessions independently completed their audio, transcript, response, playback, and close lifecycles. This is isolation and admission evidence for the tested two-session deployment, not a production fairness, arbitrary-capacity, -or failure-recovery claim. The validated paths include the experimental runtime -extension, typed resumable policy, extracted control plane and clients, request +or failure-recovery claim. The validated paths include the MiniCPM duplex +runtime extension, typed resumable policy, extracted control plane and clients, request preregistration, typed direct-output decision, engine-managed two-session capacity in the checked-in deploy profile, separate public/runtime configuration channels, private Session ledgers, ordered `session.update`, the MiniCPM runner @@ -67,7 +75,9 @@ The checkpoint does not claim: - an acoustic-onset-to-cancel latency target across arbitrary client chunk sizes; - production multi-session admission, fairness, capacity, or failure recovery; - bounded long-session KV; -- video input or audio/video synchronization. +- audio/video synchronization beyond one camera frame per model unit (camera + frames ride the audio appends as `video_frames`; the wire contract is in the + [Realtime Duplex API](../serving/realtime_duplex_api.md) guide). ## Active Runtime Path @@ -81,7 +91,7 @@ WebSocket -> AsyncOmni thin open/append/signal/close proxies -> DuplexControlClient over the engine-owned correlated RPC transport -> DuplexControlPlane + DuplexSessionRuntimeManager - -> experimental DuplexRuntimeExtension + engine session/stage bindings + -> MiniCPM DuplexRuntimeExtension + engine session/stage bindings -> StagePool -> resumable scheduler request -> MiniCPM Stage0 @@ -177,21 +187,21 @@ ordering guarantee. ### Extracted control ownership -The generic engine mechanism and the experimental duplex plugin are split by +The generic engine mechanism and the duplex plugin are split by responsibility: | Module | Responsibility | | --- | --- | | `engine.messages` | ordinary engine queue messages and collective RPC envelopes | | `engine.rpc_result_router` | generic correlation routing through each result message's `rpc_correlation_key` | -| `experimental.fullduplex.engine.contracts` | immutable duplex DTOs, extension/stage protocols, typed decisions, request-ID codec | -| `experimental.fullduplex.engine.lease` | duplex lease activity, TTL/grace configuration, and expiry records | -| `experimental.fullduplex.engine.messages` | identity fence and duplex control message/result envelopes | -| `experimental.fullduplex.engine.duplex_runtime` | extension loading, validation, and compatibility exports | -| `experimental.fullduplex.engine.duplex_session` | session resources, fences, append reservations, idempotency cache, binding cleanup | -| `experimental.fullduplex.engine.duplex_control_plane` | control algorithms and the narrow `DuplexStagePort` used to submit and clean up stage requests | -| `experimental.fullduplex.engine.duplex_control_client` | typed control-message construction and calls through the engine-owned correlated RPC transport | -| `experimental.fullduplex.request_client` | deterministic request identity, client-state preregistration, acknowledgement validation, data-plane collection, rollback | +| `engine.duplex.contracts` | immutable duplex DTOs, extension/stage protocols, typed decisions, request-ID codec | +| `engine.duplex.lease` | duplex lease activity, TTL/grace configuration, and expiry records | +| `engine.duplex.messages` | identity fence and duplex control message/result envelopes | +| `engine.duplex.runtime` | extension loading, validation, and compatibility exports | +| `engine.duplex.session` | session resources, fences, append reservations, idempotency cache, binding cleanup | +| `engine.duplex.control_plane` | control algorithms and the narrow `DuplexStagePort` used to submit and clean up stage requests | +| `engine.duplex.control_client` | typed control-message construction and calls through the engine-owned correlated RPC transport | +| `entrypoints.duplex_request_client` | deterministic request identity, client-state preregistration, acknowledgement validation, data-plane collection, rollback | `Orchestrator`, `AsyncOmniEngine`, and `AsyncOmni` retain explicit wiring and compatibility proxies. They no longer own the corresponding duplex algorithms. @@ -362,33 +372,34 @@ vllm_omni.engine.rpc_result_router # generic correlation routing vllm_omni.engine.async_engine_utils # ordinary engine lifecycle helpers ``` -The full-duplex implementation remains experimental: +The full-duplex implementation lives in dedicated duplex modules: ```text -vllm_omni.experimental.fullduplex.engine.duplex_runtime +vllm_omni.engine.duplex.runtime # extension loading and validation -vllm_omni.experimental.fullduplex.engine.contracts +vllm_omni.engine.duplex.contracts # immutable DTOs and narrow protocols -vllm_omni.experimental.fullduplex.engine.lease +vllm_omni.engine.duplex.lease # duplex lease primitives -vllm_omni.experimental.fullduplex.engine.messages +vllm_omni.engine.duplex.messages # duplex identity and queue envelopes -vllm_omni.experimental.fullduplex.engine.duplex_session +vllm_omni.engine.duplex.session # session transaction implementation -vllm_omni.experimental.fullduplex.engine.duplex_control_plane +vllm_omni.engine.duplex.control_plane # engine-side control algorithms -vllm_omni.experimental.fullduplex.engine.duplex_control_client +vllm_omni.engine.duplex.control_client # correlated control client -vllm_omni.experimental.fullduplex.request_client +vllm_omni.entrypoints.duplex_request_client # entrypoint request/output lifecycle ``` -Stable engine and entrypoint modules import only the kernel at module load. -They load the experimental ControlPlane, ControlClient, request client, and -runtime extension inside duplex-only construction or call paths. Importing -`Orchestrator`, `AsyncOmniEngine`, or `AsyncOmni` for an ordinary deployment -therefore does not import `vllm_omni.experimental.fullduplex`. Compatibility -exports remain lazy and are not part of ordinary startup. +Stable engine and entrypoint modules import the duplex kernel eagerly at +module load (top-of-file imports). Duplex behavior remains opt-in: the +ControlPlane, ControlClient, and request client are *constructed* only on +duplex-enabled deployments, and the model runtime extension still loads only +through the dotted-string plugin path. The client-side package +(`vllm_omni/clients/`, home of the public `DuplexClient`) is never imported +by runtime code. `PipelineConfig.duplex_control_enabled` enables the generic mechanism, while `PipelineConfig.duplex_runtime_extension` separately selects the model adapter. @@ -452,7 +463,7 @@ sampling generation unchanged. `extra_body` or session payloads: | Field | Default | Owner | Lifecycle or resource effect | -|---|---:|---|---| +| --- | ---: | --- | --- | | `idle_ttl_s` | `300.0` | Engine lease manager | Expires an attached or detached session after inactivity; `null` disables only idle expiry. | | `disconnect_grace_s` | `30.0` | Engine lease manager and API attachment registry | Bounds how long a detached session may retain engine resources before cleanup. | | `reaper_interval_s` | `5.0` | Orchestrator | Sets the cadence for lease expiry and pending cleanup retries. | @@ -467,6 +478,30 @@ Deploy profiles may lower or raise these values to match scheduler capacity, but clients cannot override them. The engine remains authoritative for admission and cleanup ownership even when the API performs an early check. +### Client-side companion package + +The public client surface ships under `vllm_omni/clients/` and stays on the +consumer side of the boundary above: `vllm_omni.clients.duplex` provides the +model-agnostic `DuplexClient` (typed events, per-response demux via +`ResponseHandle`, `session.resume` with acknowledgement-based replay dedup) +and the `EventCollector` helper used for timing and request-metric summaries. +Model-specific session defaults live in thin per-model preset modules — +`vllm_omni.clients.minicpmo_4_5.create_duplex_session_config()` and +`vllm_omni.clients.personaplex.create_duplex_session_config()` — so the core +client never imports model packages and the model duplex plugin packages +never import the client (both directions are enforced by tests: +`tests/engine/test_duplex_import_boundary.py`, +`tests/entrypoints/openai_api/test_duplex_capability.py`, and the subprocess +purity guards in `tests/clients/test_model_session_configs.py`). +`examples/online_serving/barge_in_client.py` (with its flow companion +`barge_in_client_flow.md`) is the runnable reference for the client library +against a live duplex deployment. The wire vocabulary the client speaks — +every `/v1/realtime?duplex=1` event with its OpenAI-compatibility tier and a +JSON example, plus the native-to-Realtime name map — is catalogued in the +[Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide; that +page derives from the normative contract table above and must be updated +with it. + ## Generic-Path Cleanup Model-specific `MINICPMO45_PROFILE_LOGS` probes were removed from: @@ -508,14 +543,16 @@ The MiniCPM Stage1 input processor consumes that capability for token-faithful delta transcripts. The generic orchestrator no longer installs or names a MiniCPM-specific bridge-state key. -The stable API server inspects the deployment `session_mode` before importing or -constructing the experimental duplex handler. Ordinary deployments do not load -the full-duplex serving package or create its registries and background state. +The stable API server imports the duplex serving package eagerly and inspects +the deployment `session_mode` before constructing the duplex handler. Ordinary +deployments do not create its registries and background state. For an enabled deployment, the client still selects the Realtime duplex route with `?duplex=1` (or an equivalent explicit true value). Model-name matching is not used for routing or native-runtime activation. MiniCPM clients explicitly -set `extra_body.minicpmo45_native_duplex=true`; repository demos do so in their -session payloads. +set `extra_body.native_duplex=true`; the canonical way to do so is +the `vllm_omni.clients.minicpmo_4_5.create_duplex_session_config()` preset, +which the repository's barge-in example +(`examples/online_serving/barge_in_client.py`) uses in its session payload. `DuplexRequestClient` derives the data-plane request identity from the accepted fence and preregisters a resumable `ClientRequestState` before the append crosses @@ -623,7 +660,7 @@ continues to return `input_audio_buffer_empty`. The commit-to-response contract is: | Session state at `input_audio_buffer.commit` | Runtime action | Required event outcome | -|---|---|---| +| --- | --- | --- | | Auto-response disabled, valid idle input | Commit and retain the input item. | Emit `input_audio_buffer.committed`; emit no response until an explicit `response.create`. | | Auto-response enabled, valid idle speech | Submit the final native append for the current model turn. | Emit `input_audio_buffer.committed` before model output. Create a response only on the first visible text/audio output; a model listen decision may end with `response.listen` and no response. | | Auto-response enabled, short overlap during an active response | Discard the short overlap as an acknowledgement. | Emit the committed/no-response acknowledgement; do not create or defer a response. | @@ -770,9 +807,12 @@ Graph capture. This is a model execution constraint, not an E2E-only override. ### Resource capabilities -The checked-in MiniCPM duplex deploy profile sets Stage0 and Stage1 -`max_num_seqs` to two and configures the engine runtime manager with -`max_sessions=2`. The Realtime capability response therefore advertises +At the time of this snapshot the checked-in MiniCPM duplex deploy profile set +Stage0 and Stage1 `max_num_seqs` to two and configured the engine runtime +manager with `max_sessions=2`; the duplex settings have since been folded into +`vllm_omni/deploy/minicpmo_4_5.yaml`, which admits four sessions +(`duplex_session.max_sessions: 4`, matching its stage `max_num_seqs`). The +Realtime capability response advertises multi-session and same-replica multi-session support with `session_admission_mode="engine_managed"`. Client session fields cannot raise this server-owned limit. Fresh E2E evidence covers two concurrent sessions in @@ -1025,4 +1065,6 @@ Passing this checkpoint supports the statement: It does not support claims for an acoustic-onset barge-in latency target across arbitrary client chunk sizes, multi-session production concurrency, bounded -long-session KV, scheduler-native append, or video input. +long-session KV, or scheduler-native append. Camera-frame input +(`video_frames`) landed after this snapshot and is covered by its own E2E +test rather than by the evidence above. diff --git a/docs/design/index.md b/docs/design/index.md index 066d257d1..81eac4736 100644 --- a/docs/design/index.md +++ b/docs/design/index.md @@ -17,6 +17,8 @@ implementation contract; it is not, by itself, a general support claim. ### Runtime and stage execution +- [Full-Duplex Runtime (MiniCPM-o 4.5)](fullduplex.md) +- [Full-Duplex Runtime (PersonaPlex)](fullduplex-personaplex.md) - [Disaggregated Inference](feature/disaggregated_inference.md) - [Host Weight Runtime](feature/host_weight_runtime.md) - [Async Chunk](feature/async_chunk.md) diff --git a/docs/design/module/engine_orchestration.md b/docs/design/module/engine_orchestration.md index f3f07a43f..d4dbf51a2 100644 --- a/docs/design/module/engine_orchestration.md +++ b/docs/design/module/engine_orchestration.md @@ -28,6 +28,9 @@ related_code_paths: - vllm_omni/engine/stage_pool.py - vllm_omni/outputs/output_processor.py - vllm_omni/distributed/omni_coordinator/** + # Duplex kernel design is owned by ../fullduplex.md; listed here because the + # orchestrator hosts the duplex control plane. + - vllm_omni/engine/duplex/** depends_on: - input_output_modality_contracts.md - error_contracts.md @@ -44,7 +47,8 @@ validation_paths: - tests/engine/test_orchestrator_stage_input_bridge.py - tests/engine/test_cfg_companion_lifecycle.py - tests/engine/test_rpc_result_router.py - - tests/e2e/features/fullduplex/engine/** + - tests/engine/duplex/** + - tests/engine/test_duplex_import_boundary.py upstream_refs: - vllm.v1.engine.EngineCoreRequest - vllm.v1.engine.EngineCoreOutputs diff --git a/docs/design/module/entrypoints.md b/docs/design/module/entrypoints.md index fcdfe23ea..a0dee90b9 100644 --- a/docs/design/module/entrypoints.md +++ b/docs/design/module/entrypoints.md @@ -26,9 +26,18 @@ primary_code_paths: - vllm_omni/entrypoints/client_request_state.py - vllm_omni/entrypoints/stage_utils.py - vllm_omni/entrypoints/utils.py + - vllm_omni/entrypoints/duplex/** + - vllm_omni/entrypoints/duplex_request_client.py + - vllm_omni/clients/** primary_path_exceptions: - path: vllm_omni/entrypoints/openai/errors.py owner: error_contracts.md + - path: vllm_omni/entrypoints/duplex/** + owner: ../fullduplex.md + - path: vllm_omni/entrypoints/duplex_request_client.py + owner: ../fullduplex.md + - path: vllm_omni/clients/** + owner: ../fullduplex.md related_code_paths: - vllm_omni/errors.py - vllm_omni/inputs/** diff --git a/docs/mkdocs/hooks/generate_api_readme.py b/docs/mkdocs/hooks/generate_api_readme.py index e09c6e1ea..3130e688f 100644 --- a/docs/mkdocs/hooks/generate_api_readme.py +++ b/docs/mkdocs/hooks/generate_api_readme.py @@ -22,6 +22,13 @@ CATEGORIES = { "name": "Entry Points", "description": "Main entry points for vLLM-Omni inference and serving.", }, + "clients": { + "name": "Clients", + "description": ( + "Client libraries that talk to a running vLLM-Omni server " + "(the full-duplex Realtime client and its per-model session presets)." + ), + }, "inputs": { "name": "Inputs", "description": "Input data structures for multi-modal inputs.", diff --git a/docs/serving/README.md b/docs/serving/README.md index 6acfa534a..8167f4c9b 100644 --- a/docs/serving/README.md +++ b/docs/serving/README.md @@ -43,7 +43,7 @@ Completions for conversational or heterogeneous omni pipelines, rather than as the default wrapper for every generation task. | Task | Endpoint | Request | Response | Details | -|------|----------|---------|----------|---------| +| ------ | ---------- | --------- | ---------- | --------- | | Conversation or multimodal understanding/generation | `POST /v1/chat/completions` | JSON | OpenAI-style JSON or SSE | [Chat Completions](chat_completions_api.md) | | Text-to-speech | `POST /v1/audio/speech` | JSON | Audio bytes or SSE | [Speech](speech_api.md) | | Sound, music, or ambient audio generation | `POST /v1/audio/generate` | JSON | Audio bytes | [Audio Generation](audio_generate_api.md) | @@ -141,7 +141,7 @@ These WebSocket APIs have different event schemas and cannot be used interchangeably. | Workload | Endpoint | Interaction model | Details | -|----------|----------|-------------------|---------| +| ---------- | ---------- | ------------------- | --------- | | Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) | | Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) | | Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) | @@ -158,7 +158,7 @@ ordinary realtime handler when duplex is unavailable. ## Related Endpoints | Purpose | Endpoints | Reference | -|---------|-----------|-----------| +| --------- | ----------- | ----------- | | Discovery and readiness | `GET /health`, `GET /v1/models` | This page | | Batched conversations | `POST /v1/chat/completions/batch` | [Batch requests](chat_completions_api.md#batch-requests) | | Batched speech | `POST /v1/audio/speech/batch` | [Batch speech generation](speech_api.md#batch-speech-generation) | @@ -168,11 +168,10 @@ ordinary realtime handler when duplex is unavailable. ## Standalone Experimental Servers -PersonaPlex also provides a Moshi-compatible WebSocket server, while JoyVL -provides a stateful interaction orchestrator in front of another model server. -These are separate processes with their own routes; they are not additional +JoyVL provides a stateful interaction orchestrator in front of another model +server. It is a separate process with its own routes; they are not additional paths on every `vllm serve --omni` instance. See [Standalone Experimental -Servers](standalone_servers.md) before deploying either one. +Servers](standalone_servers.md) before deploying it. ## Compatibility Notes diff --git a/docs/serving/full_duplex_api.md b/docs/serving/full_duplex_api.md index c7c2ac44c..f648d6fdb 100644 --- a/docs/serving/full_duplex_api.md +++ b/docs/serving/full_duplex_api.md @@ -1,16 +1,20 @@ # Full-Duplex WebSocket API -vLLM-Omni provides an experimental full-duplex runtime for models that can +vLLM-Omni provides a full-duplex runtime for models that can continue receiving speech while producing speech. It adds persistent session state, model-specific turn policy, overlap handling, playback acknowledgement, and optional session resume. Full duplex is distinct from the turn-based [Realtime Audio API](realtime_api.md). +This page is the endpoint overview. For the complete wire contract, the +`vllm_omni.clients.duplex.DuplexClient` Python library, and the per-model +capability gates, see the [Realtime Duplex API guide](realtime_duplex_api.md). + ## Choose an Endpoint | Endpoint | Protocol | Recommended use | -|----------|----------|-----------------| +| --- | --- | --- | | `WS /v1/realtime?duplex=1` | OpenAI Realtime-style event projection | Applications and browser clients | | `WS /v1/duplex` | Native vLLM-Omni duplex events | Runtime integration and low-level testing | @@ -41,8 +45,10 @@ and control plane as part of their registered pipeline. The current unified-runtime integrations are: -- MiniCPM-o 4.5, using `vllm_omni/deploy/minicpmo_4_5_duplex.yaml`; -- PersonaPlex, whose default `vllm_omni/deploy/personaplex.yaml` enables duplex. +- MiniCPM-o 4.5, using `vllm_omni/deploy/minicpmo_4_5.yaml` + (`session_mode: duplex`); +- PersonaPlex, whose default `vllm_omni/deploy/personaplex.yaml` enables duplex; +- Nemotron VoiceChat, via its registered duplex plugin package. JoyVL is a separate HTTP interaction orchestrator and does not use these WebSocket endpoints. See [Standalone Experimental Servers](standalone_servers.md). @@ -53,7 +59,7 @@ Start the duplex deployment: ```bash vllm serve openbmb/MiniCPM-o-4_5 --omni \ - --deploy-config vllm_omni/deploy/minicpmo_4_5_duplex.yaml \ + --deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \ --trust-remote-code \ --port 8091 ``` @@ -112,5 +118,5 @@ applications should use the provided Realtime client where possible. See the [MiniCPM-o example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/minicpmo), [PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex), -and [full-duplex runtime design](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/experimental/fullduplex/DESIGN.md) +and the [full-duplex runtime design](../design/fullduplex.md) for model-specific validation and architecture details. diff --git a/docs/serving/realtime_api.md b/docs/serving/realtime_api.md index 58fe8b7c3..153564140 100644 --- a/docs/serving/realtime_api.md +++ b/docs/serving/realtime_api.md @@ -5,7 +5,7 @@ receive incremental transcription and synthesized response audio. This is a turn-based streaming API: it starts one generation over an audio stream and finishes after the client closes that input stream. -For a model that listens while it is already speaking, use the experimental +For a model that listens while it is already speaking, use the [Full-Duplex API](full_duplex_api.md) instead. ## Quick Start @@ -35,7 +35,7 @@ Messages in both directions are JSON text frames. Audio carried inside an event is base64-encoded raw PCM16. | Direction | Event | Purpose | -|-----------|-------|---------| +| ----------- | ------- | --------- | | Server to client | `session.created` | Confirms that the WebSocket connection is ready | | Client to server | `session.update` | Selects and validates the served model | | Client to server | `input_audio_buffer.commit` with `final: false` | Starts generation over the incoming stream | diff --git a/docs/serving/realtime_duplex_api.md b/docs/serving/realtime_duplex_api.md new file mode 100644 index 000000000..30021e107 --- /dev/null +++ b/docs/serving/realtime_duplex_api.md @@ -0,0 +1,1217 @@ +# Realtime Duplex API + +vLLM-Omni serves full-duplex speech models over a WebSocket endpoint, +`/v1/realtime?duplex=1`, that speaks the OpenAI Realtime vocabulary plus a +set of duplex extensions: the model decides when to listen and when to speak, +the user can talk over the assistant, playback progress is reported back so +history stays honest, and a dropped connection can resume the same session. +This page covers how to run a duplex deployment, how to drive it from Python +with `vllm_omni.clients.duplex.DuplexClient`, and the complete wire contract. + +The endpoint is served for models that ship a duplex plugin — currently +MiniCPM-o 4.5, PersonaPlex, and Nemotron VoiceChat — and only for deploy +configurations that declare `session_mode: duplex`. The runtime architecture +is described in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md). + +## Quick Start + +### Start the Server + +```bash +vllm-omni serve openbmb/MiniCPM-o-4_5 \ + --omni \ + --deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \ + --trust-remote-code \ + --host 0.0.0.0 --port 8099 +``` + +`vllm_omni/deploy/minicpmo_4_5.yaml` declares `session_mode: duplex` and +`duplex_session.max_sessions: 4`, which mounts the WebSocket routes +`ws://:8099/v1/realtime?duplex=1` (this page) and `ws://:8099/v1/duplex` +(the native dialect for raw-protocol clients) next to the usual HTTP API. +PersonaPlex and Nemotron VoiceChat use `vllm_omni/deploy/personaplex.yaml` and +`vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml` in the same way. + +### Run the Example Client + +`examples/online_serving/barge_in_client.py` asks a question, interrupts the +answer mid-stream, and writes one WAV per response: + +```bash +python examples/online_serving/barge_in_client.py \ + --url ws://127.0.0.1:8099/v1/realtime \ + --model openbmb/MiniCPM-o-4_5 \ + --ref-audio /path/to/reference_voice.wav \ + --question-wav question_16k.wav \ + --interrupt-wav follow_up_16k.wav \ + --output-dir ./duplex_out +``` + +Input WAVs must be mono 16 kHz PCM16. `--ref-audio` is required by the +MiniCPM-o preset (the session is otherwise rejected with +`ref_audio_required`). `--preset personaplex` switches the session shape to +the PersonaPlex preset. The flow is diagrammed in +`examples/online_serving/barge_in_client_flow.md`. + +## Using DuplexClient + +`vllm_omni.clients.duplex` is a client-side package: it depends only on the +standard library, `pybase64`, and the `websockets` package, and it is never +imported by server code. Full signatures are on the +[API reference page](../api/README.md#clients). + +### Open a session + +`DuplexClient` is an async context manager. Entering it connects, sends +`session.update`, and waits for `session.created`; exiting sends +`session.close` and waits for `session.closed`. + +```python +from vllm_omni.clients.duplex import DuplexClient, audio_data_url +from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config + +config = create_duplex_session_config(ref_audio=audio_data_url(Path("reference_voice.wav"))) +async with DuplexClient("ws://127.0.0.1:8099/v1/realtime", model="openbmb/MiniCPM-o-4_5", config=config) as client: + print(client.session_id, client.session_info["capabilities"]) +``` + +The session shape comes from `SessionConfig`; model-specific knobs ride in +its `extra_body`, so the client itself stays model-neutral and each model +ships a preset: + +| Preset | Input / output audio | What it sets | +| --- | --- | --- | +| `vllm_omni.clients.minicpmo_4_5.create_duplex_session_config(ref_audio=...)` | 16 kHz `pcm16` / 24 kHz `pcm16` | `extra_body.native_duplex=True` (opt in to the model-native lane), `force_listen_count=0`, `overlap_policy="listen_only"`, `playback_commit_policy="ack_only"`; `ref_audio` is the assistant voice clip | +| `vllm_omni.clients.personaplex.create_duplex_session_config(voice="NATF2.pt", persona="")` | 24 kHz `pcm_f32le` / 24 kHz `pcm16` | bundled `.pt` voice prompt and the persona as `instructions` | +| `SessionConfig(...)` | 16 kHz `pcm16` / 24 kHz `pcm16` | model-neutral defaults; pass `extra_body`, `turn_detection`, `overlap_policy`, `playback_commit_policy`, `instructions`, `voice`, `temperature` yourself | + +Keyword overrides on a preset replace the corresponding `SessionConfig` +field, e.g. `create_duplex_session_config(ref_audio=..., temperature=0.6)`. + +Constructor options: `session_id` (names the session the client creates; it +does not resume or take over an existing session — entering the client always +performs the `session.update` handshake), `reconnect` (a `ReconnectPolicy`, +default five attempts with 0.25–4 s jittered backoff; `None` disables +auto-resume), `heartbeat_interval_s` (default 30 s; `None` disables the lease +heartbeat), `handshake_timeout_s`, and `connect` (a custom transport factory, +used by tests and benchmarks). Resume is supported only as automatic reconnect +within the same `DuplexClient` instance (see +[Reconnect and resume](#reconnect-and-resume)); re-attaching from a new client +or process requires the wire-level `session.resume` handshake +(`resume_token`, `incarnation`, `last_received_server_event_seq`), which this +client does not expose. + +### Stream audio and commit a turn + +```python +from vllm_omni.clients.duplex import read_pcm16_wav + +question = read_pcm16_wav(Path("question_16k.wav")) +await client.stream_pcm(question, chunk_ms=100) # paced 100 ms appends +await client.commit(create_response=False) # the model decides listen/speak +``` + +`stream_pcm` slices PCM into `chunk_ms` appends and paces them in real time +(`realtime=False` sends as fast as possible). `append_audio` sends one chunk +and accepts `is_speech` and `video_frames` (base64 JPEG/PNG strings riding +the append that closes a one-second model unit — data URLs are accepted and +their prefix stripped; `stream_pcm(video_frames=..., +stacked_video_frames=...)` interleaves a frame track automatically and +returns the number of frames sent). A turn is ended with `commit`, never by +a flag on the append. + +`commit` seals the buffered utterance into a user item. With the default +`auto_response=True` a commit lets the model respond; pass +`create_response=False` to leave the listen/speak decision entirely to a +model-native session, or set `auto_response=False` in the config and call +`client.send({"type": "response.create"})` when you want an answer. In the +model-native lane the audio is already streaming into the model before the +commit, so the model may start answering — or emit a listen decision — +without any commit at all. + +### Consume responses + +`responses()` demultiplexes the event stream into one `ResponseHandle` per +assistant response. A model listen decision surfaces as an already-finished +handle with `decision == "listen"`. + +```python +async for response in client.responses(): + if response.decision == "listen": + continue # the model kept listening + async for chunk in response.audio(): # decoded PCM16 at 24 kHz + play(chunk) + await client.ack_playback(response.played_ms, response_id=response.response_id) + await response.wait() + print(response.transcript, response.done_event.raw["response"]["status"]) + break +``` + +`ResponseHandle` exposes `response_id`, `decision` (`"speak"` / `"listen"`), +the accumulated `transcript` and `text`, `played_ms`, `created_event`, +`done_event`, `finished`, `audio()` and `wait()`. + +`ack_playback(played_ms, response_id=..., committed_ms=...)` reports the +cumulative playback position. Call it while audio plays: with the MiniCPM-o +preset's `playback_commit_policy="ack_only"` the assistant turn enters the +conversation history only as far as you acknowledge it, which is what keeps +history truthful after an interruption. `acknowledge_collected_playback(client, +collector)` acknowledges everything an `EventCollector` has received, which is +what the example and the benchmarks do at the end of a turn. + +### Interrupt + +```python +await client.cancel_response() # cancel the active response (epoch advances) +await client.clear_input() # drop un-committed input audio +``` + +In the model-native lane you usually do not need either: keep streaming +microphone audio while the assistant speaks and the server's overlap policy +decides — a short remark is acknowledged, meaningful overlap is deferred to +the turn end, and on models that advertise `supports_barge_in` (MiniCPM-o +4.5) the in-flight response is cancelled. Each decision is reported as an +`overlap.decision` event. `cancel_response` and `clear_input` compose a +client-forced barge-in for clients that run their own VAD. + +### Work with raw events + +`events()` (or `async for event in client`) yields every server event from +now on as a typed `DuplexEvent` (`SessionCreated`, `ResponseCreated`, +`AudioDelta`, `TranscriptDelta`, `TextDelta`, `ListenDecision`, +`SpeakDecision`, `ResponseDone`, `SessionClosed`, `SessionExpired`, +`ErrorEvent`, and `ConnectionResumed` after an automatic resume; unknown +types arrive as plain `DuplexEvent`). Common accessors are `type`, +`event_id`, `session_id`, `response_id`, `item_id`, `server_event_seq`, +`audio` (decoded bytes), `text`, and `raw` (the wire dict). +`wait_for("response.done", timeout_s=30)` blocks for the next event of the +given type(s), and `send({...})` is the escape hatch for any client event +the wrapper does not have a method for (`response.create`, +`conversation.item.create`, `conversation.item.truncate`, ...); it returns +the stamped `event_id`. + +### Reconnect and resume + +On a transport drop the client reconnects with the `ReconnectPolicy` backoff +and sends `session.resume` from the last acknowledged `server_event_seq`. +Replayed events are deduplicated, consumers see one `ConnectionResumed` +event, and `client.incarnation` / `client.resume_token` are refreshed. The +server keeps a detached session alive for `disconnect_grace_s` (30 s by +default); after that a resume fails with `session_resume_expired`. Resume is +gated by `capabilities.supports_session_resume` (MiniCPM-o 4.5 and Nemotron +VoiceChat: yes; PersonaPlex: no). With `reconnect=None` the same drop surfaces +as `DuplexSessionClosedError`. + +### Errors + +Server `error` events are delivered as `ErrorEvent` (with `code`, `message`, +`related_event_id`) and raised as `DuplexProtocolError` from the awaiting +call; transport failures raise `DuplexConnectionError`; a session that ends +while you are awaiting it raises `DuplexSessionClosedError`. All three derive +from `DuplexClientError`. + +### Measure latency + +`EventCollector` accumulates events for assertions and metrics: + +```python +from vllm_omni.clients.duplex import EventCollector + +collector = EventCollector() +consume_task = asyncio.create_task(collector.consume(client)) +commit_sent_at = time.monotonic() +await client.commit(create_response=False) +... +summary = collector.timing_summary(after_s=commit_sent_at, input_committed_at_s=commit_sent_at) +``` + +`timing_summary` reports client-observed first-text / first-audio latency, +audio cadence, and, when the deployment attaches engine stage metrics to the +response events (the benchmark requests them with +`extra_body.return_stage_metrics=True`), the Stage 0 token metrics +(`ttft_ms`, `tpot_ms`, `itls_ms`). It reports raw measurements only — +derived metrics such as the audio RTF are computed by the caller from +`audio_generation_ms` / `audio_duration_ms` (the benchmarks use +`vllm_omni.metrics.definitions.compute_audio_rtf`), keeping metric +definitions out of the dependency-free client. `audio_bytes(response_id)`, `count(type)`, +`errors()`, and `first_received_at(type)` cover the common assertions; the +`openai-realtime-duplex` benchmark backend (`vllm-omni bench serve`) is built +on the same helpers. + +## API Reference + +This section is the complete wire contract of `/v1/realtime?duplex=1`: how +the dialect relates to the OpenAI Realtime protocol, which surfaces each +model's capability flags gate, a catalogue of every client-to-server and +server-to-client event, and a JSON example for each. `DuplexClient` speaks +exactly this contract; raw WebSocket clients (any OpenAI Realtime client, or +your own) can use it directly. The ordering and cardinality rules behind it +are specified in the normative contract section of +[Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md); a +compatibility change must update this section together with that table and +its protocol tests. + +### Endpoint and transport + +The endpoint is `ws(s):///v1/realtime?duplex=1`; the `duplex` query +parameter (`1`, `true`, or `on`) selects the duplex session handler, and the +route is mounted only for deployments whose deploy configuration declares +`session_mode: duplex`. Optional query parameters are `model`, `session_id`, +`autostart` (`0` means resume-only), `resume`, and `native_duplex` +(`minicpmo45_native_duplex` is accepted as a deprecated alias and folded +into the canonical name). Every message is one JSON object per WebSocket +text frame, discriminated by `type`. Inbound events are applied in arrival +order through one per-session mailbox; outbound events preserve that order +and carry a monotonically increasing `server_event_seq`, which is the replay +cursor for `session.resume`. Client events may carry an `event_id`, echoed in +`error.error.event_id` when the event is rejected. + +Session identity is `(session_id, incarnation)`; `attachment_generation` +names the current socket; `epoch` is the interruption fence that advances on +every cancel, clear, or barge-in; `turn_id` counts committed model turns; +`response_id` (item id `item_`) names one assistant utterance. + +**Tier** in the tables below: **1** means identical to OpenAI Realtime in +name and semantics; **2** means an OpenAI event name carrying vLLM-Omni +extensions that a stock OpenAI client ignores; **3** means vLLM-Omni only, +with no OpenAI counterpart. + +### Capability negotiation by model + +The event vocabulary is uniform, but several surfaces are gated by the +`capabilities` object the server returns in `session.created`; a client must +branch on those flags rather than on the model name. The current model +plugins advertise: + +| Capability | MiniCPM-o 4.5 | PersonaPlex | Nemotron VoiceChat | Gated surface | +| --- | --- | --- | --- | --- | +| `implementation_level` | `model_native_duplex` (when `extra_body.native_duplex=true`) | `model_native_duplex` | `model_native_duplex` | model-owned `response.listen` / `response.speak`; the chat-fallback lane otherwise | +| `chunk_period_ms` | 1000 | 80 | 80 | the model unit that `response.listen` decisions and camera frames align to | +| `supports_session_resume` | yes | no | yes | `session.resume`, `session.resumed`, `session.replaced`, replay after a transport drop | +| `supports_barge_in` | yes | no | no | `barge_in`, `turn.signal{event:"barge_in"}`, `overlap_policy=barge_in_on_speech`, `turn_detection.server_vad` | +| `supports_audio_truncate` | yes | no | no | `conversation.item.truncate` and truncating `playback.ack` adjusting the stored assistant item | +| video input (`video_frames` on append) | consumed by Stage 0 | ignored | ignored | omni camera track | +| tool calls | no | no | yes | `response.function_call_arguments.*`, `function_call` items | + +Everything else in the catalogue — session lifecycle, heartbeat and event +acknowledgement, append/commit/clear, the response envelope, playback +acknowledgement, and the error envelope — behaves identically for every +model. + +### Compatibility with the OpenAI Realtime protocol + +The Realtime dialect is, by design, an OpenAI-Realtime-compatible surface: +`RealtimeInputTranslator` / `RealtimeOutputProjector` exist to translate +the native duplex vocabulary onto OpenAI's schema so stock Realtime clients +work unmodified. The overlap falls into the three tiers used by the event catalogue below. + +#### Tier 1: identical to OpenAI Realtime + +| Direction | Messages | +| --- | --- | +| Client → server | `session.update`, `input_audio_buffer.append` / `.commit` / `.clear`, `conversation.item.create` / `.retrieve` / `.truncate` / `.delete`, `response.create`, `response.cancel`, `output_audio_buffer.clear` (OpenAI: WebRTC-only; here also a WebSocket event) | +| Server → client | `error` (same `{type, code, message, event_id, param}` envelope and the same classes `invalid_request_error` / `server_error` / `rate_limit_error`), `session.created`, `session.updated`, `input_audio_buffer.committed` / `.cleared` / `.speech_started` / `.speech_stopped`, `conversation.item.created` / `.added` / `.done` / `.deleted` / `.retrieved` / `.truncated`, `conversation.item.input_audio_transcription.completed`, `response.created`, `response.done`, `response.output_item.added` / `.done`, `response.content_part.added` / `.done`, `response.audio.delta` / `.done`, `response.audio_transcript.delta` / `.done`, `response.output_text.delta` / `.done`, `response.function_call_arguments.delta` / `.done`, `rate_limits.updated`, `output_audio_buffer.cleared` | +| Session fields | `model`, `modalities` / `output_modalities`, `instructions`, `voice`, `input_audio_format` / `output_audio_format` (beta spelling) **and** `audio.input.format` / `audio.output.format` objects (GA spelling), `turn_detection.server_vad {threshold, prefix_padding_ms, silence_duration_ms}`, `input_audio_transcription`, `input_audio_noise_reduction`, `tools`, `tool_choice`, `temperature`, `max_response_output_tokens`, `speed`, `tracing` | +| Object shapes | `realtime.session`, `realtime.item`, `realtime.response`; item content parts `input_text` / `input_audio` / `output_text` / `output_audio`; `function_call` and `function_call_output` items; `previous_item_id` chaining; `output_index` / `content_index` addressing | +| Sequencing | `response.created → output_item.added → content_part.added → deltas → *.done → content_part.done → output_item.done → response.done → rate_limits.updated`; `speech_started → speech_stopped → committed` | + +The server deliberately emits and accepts **both** the beta and the GA +spellings at once (`response.audio.delta` alongside `output_modalities`, +`conversation.item.added` alongside `conversation.item.created`) so either +generation of OpenAI client parses the stream. + +#### Tier 2: OpenAI names carrying vLLM-Omni extensions + +A stock client ignores the extra keys; the extensions are additive. + +| Message | Extension | +| --- | --- | +| `session.created` / `session.updated` | top-level `incarnation`, `attachment_generation`, `resume_token`; inside `session`: `state`, `turn_state`, `epoch`, `turn_id`, `active_request_id`, `active_response_id`, `active_response_turn_id`, `overlap_policy`, `overlap_*_ms/rms`, `playback_commit_policy`, `playback`, `capabilities`, `ref_audio`, `extra_body`, `idle_timeout_s`, `response_format` | +| `response.created` / `response.done` / `response.listen` | `response_id` at top level; the raw duplex event under `response.metadata` (`duplex_event`); `status_details.reason` uses vLLM reasons (`barge_in`, `client_cancelled`, `new_response`, …) | +| `response.audio.delta` | `format`, `sample_rate_hz`, `metadata{session_id, epoch, model_speak, end_of_turn, audio_duration_ms, audio_text_marks, playback}` | +| `response.speak` (inserted before the first delta) | not an OpenAI event, but rides the OpenAI response envelope (`response_id`, `item_id`, `output_index`, `content_index`) | +| `input_audio_buffer.append` | `is_speech`, `video_frames` (+ `max_slice_nums`), `duration_ms`, `audio_end_ms`, per-event `format` / `sample_rate_hz` | +| `input_audio_buffer.commit` | `final`, `response_create`, `is_speech:false` (silence declaration) | +| `input_audio_buffer.committed` | the native `input.committed` event wrapped under `event` (`turn_id`, `epoch`, `history_len`, `message`, `response_create_deferred`) | +| `conversation.item.*` server events | echo of the originating event under `event` | +| `error` | the error **codes** are vLLM-Omni's (OpenAI standardises only the class); handshake-stage errors use the flat native shape `{type:"error", error:"…", code}` | + +Semantic divergences hidden behind shared names: + +- **Commit ≠ response.** In the native lane audio streams into the model + before any commit; a commit may end in `response.listen` and no + response, and the model may open a response with no commit at all. + OpenAI: commit ⇒ item, `response.create` ⇒ exactly one response. +- `turn_detection.interrupt_response=false` is rejected; + `create_response` is ignored; `semantic_vad` is unsupported; the VAD + runs per session (Silero) and implies `overlap_policy=barge_in_on_speech`. +- `rate_limits.updated` is always an empty list (compatibility only). +- Cancellation never reuses a `response_id`; truncation is driven by + `playback.ack` as well as `conversation.item.truncate`. + +#### Tier 3: vLLM-Omni only + +| Area | Messages / fields | +| --- | --- | +| Session lifetime | `session.heartbeat` / `session.heartbeat_ack`, `session.event_ack`, `session.close` / `session.closed`, `session.resume` / `session.resumed` / `session.replaced` / `session.expired` / `session.resync_required`; `server_event_seq`, `resume_token`, `incarnation`, `attachment_generation`; query params `?duplex=1`, `autostart`, `resume`, `native_duplex` | +| Turn-taking | `response.speak`, `response.listen`, `overlap.decision`, `overlap_policy`, `barge_in`, `turn.signal`, `input.cancel`, `input.text.append`, `epoch` / `turn_id` fencing, `force_listen` | +| Playback truth | `playback.ack` / `playback.acknowledged`, `playback_commit_policy` (`ack_only` \| `commit_all_on_done`), `playback{generated_ms, sent_ms, played_ms, committed_ms}`, `history_committed`, `audio_text_marks` | +| Model negotiation | `capabilities{implementation_level, supports_input_append, supports_barge_in, supports_session_resume, chunk_period_ms, input_modes, …}`, `ref_audio`, `extra_body` (`auto_response`, `native_duplex` — deprecated alias `minicpmo45_native_duplex` is folded into it, `force_listen_count`, `duplex_initial_user_text`) | +| Diagnostics | `runtime.control`, `duplex.function_call.done`, the full error-code vocabulary (`REALTIME_ERROR_TYPES_BY_CODE` in `vllm_omni/entrypoints/duplex/realtime_state.py`) | + +#### Consequences for clients + +- Event **names** are roughly 70 % OpenAI's; payloads are OpenAI's plus + additive extensions; everything that makes the endpoint full-duplex — + model-owned turn-taking, playback truth, interruption fencing, resumable + sessions — lives in Tier 3. +- An **unmodified OpenAI Realtime client** can hold a conversation + (commit-driven or `turn_detection: server_vad`), but it will never see + `response.listen` / `response.speak`, cannot resume after a transport + drop, will not receive replay after reconnect, and its server-side + history is only as honest as `commit_all_on_done` allows because it never + sends `playback.ack`. +- A **duplex-aware client** (`vllm_omni.clients.duplex.DuplexClient`) uses + the Tier 3 surface on top of the same Tier 1 events, so both kinds of + client can share one deployment and one event log. + +### Event catalogue + +Every event type the Realtime route accepts or emits, with a worked example in the next section: 21 +client→server and 42 server→client. Aliases (`push_chunk`, +`input.audio.append`, `input_text.append`, `push_text`, `signal_turn`, +`audio.playback_ack`, `close_session`, `close`, `session_close`) share the +payload of their canonical event and are not listed separately; the alias +table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md). + +#### Client to server + +| Event type | Tier | Description | +| --- | --- | --- | +| `session.update` | 2 | Open handshake on first send (creates the session); later sends reconfigure it. OpenAI name; `session` object carries vLLM-Omni keys (`ref_audio`, `overlap_policy`, `playback_commit_policy`, `extra_body`, ...). | +| `session.heartbeat` | 3 | Keepalive; refreshes the engine session lease. | +| `session.event_ack` | 3 | Acknowledges received server events by `server_event_seq`; trims the resume replay journal. | +| `session.close` | 3 | Graceful close; answered with `session.closed`. | +| `session.resume` | 3 | Re-attach a live session on a new socket (`resume_token`, `incarnation`, `last_received_server_event_seq`). | +| `input_audio_buffer.append` | 2 | Append one audio chunk. OpenAI name; extended with `is_speech`, `video_frames`, `duration_ms`, `audio_end_ms`, per-event `format`/`sample_rate_hz`. | +| `input_audio_buffer.commit` | 2 | Seal the buffered utterance into a user item. OpenAI name; extended with `final`, `response_create`, `is_speech:false`. | +| `input.commit` | 3 | Alias of `input_audio_buffer.commit` accepted by the runner. | +| `input_audio_buffer.clear` | 1 | Drop un-committed input audio. | +| `input.cancel` | 3 | Cancel pending input; advances `epoch`. | +| `input.text.append` | 3 | Append text to the open input item (chat-fallback lane only). | +| `conversation.item.create` | 1 | Add a completed user item (text, audio, or `function_call_output`) to history. | +| `conversation.item.retrieve` | 1 | Fetch a stored item by id. | +| `conversation.item.delete` | 1 | Delete a stored item by id. | +| `conversation.item.truncate` | 1 | Truncate an assistant item's audio/transcript at `audio_end_ms`. | +| `response.create` | 1 | Explicitly request a response with per-response overrides. | +| `response.cancel` | 1 | Cancel the active (or named) response; advances `epoch`. | +| `output_audio_buffer.clear` | 1 | Discard queued output audio; advances `epoch` (OpenAI: WebRTC-only, here also WebSocket). | +| `barge_in` | 3 | Explicit hard interrupt; requires `capabilities.supports_barge_in`. | +| `turn.signal` | 3 | External turn-taking signal (e.g. `event:"barge_in"`); same capability gate. | +| `playback.ack` | 3 | Report cumulative playback progress (`played_ms`, `committed_ms`, optional `truncate`). | + +#### Server to client + +| Event type | Tier | Description | +| --- | --- | --- | +| `session.created` | 2 | Session opened. OpenAI name; adds `incarnation`, `attachment_generation`, `resume_token` and vLLM-Omni keys inside `session` (`epoch`, `turn_id`, `playback`, `capabilities`, ...). | +| `session.updated` | 2 | Echo of the effective session config after every `session.update` (same extended `session` object). | +| `session.heartbeat_ack` | 3 | Reply to `session.heartbeat`. | +| `session.closed` | 3 | Last event on the socket; `reason` ∈ `session_close`, `timeout`, `disconnect`, ... | +| `session.resumed` | 3 | Resume accepted; carries the new `attachment_generation` and rotated `resume_token`; journaled events are replayed after it. | +| `session.replaced` | 3 | Sent to the superseded socket when another socket resumes the session. | +| `session.expired` | 3 | Engine lease reaped (`disconnect_grace_expired`, `idle_ttl_expired`); socket closes after it. | +| `session.resync_required` | 3 | Replay impossible (`journal_gap` / `journal_overflow`); client must start a new session. | +| `input_audio_buffer.speech_started` | 1 | Speech onset detected (server VAD, client hint, or RMS heuristic). | +| `input_audio_buffer.speech_stopped` | 1 | Speech end detected. | +| `input_audio_buffer.committed` | 2 | Commit accepted. OpenAI name; wraps the native `input.committed` payload under `event` (`turn_id`, `epoch`, `response_create_deferred`, ...). | +| `input_audio_buffer.cleared` | 1 | Input buffer cleared (after `input_audio_buffer.clear` or `input.cancel`). | +| `conversation.item.added` | 2 | Item entered history (GA spelling). OpenAI name; carries `previous_item_id` and echoes the originating event. | +| `conversation.item.created` | 2 | Same as `conversation.item.added` in the beta spelling; both are emitted. | +| `conversation.item.done` | 2 | Item finalized (GA spelling). | +| `conversation.item.input_audio_transcription.completed` | 1 | Transcript of a committed user audio item (model-produced). | +| `conversation.item.deleted` | 2 | Reply to `conversation.item.delete` (echoes the request under `event`). | +| `conversation.item.retrieved` | 1 | Reply to `conversation.item.retrieve`. | +| `conversation.item.truncated` | 2 | Reply to `conversation.item.truncate` (echoes the request under `event`). | +| `response.created` | 2 | A visible assistant response starts. OpenAI name; top-level `response_id`, raw duplex event under `response.metadata`. | +| `response.output_item.added` | 1 | Assistant item (or `function_call` item) attached to the response. | +| `response.content_part.added` | 1 | Audio/text content part opened on the assistant item. | +| `response.speak` | 3 | Model chose to speak (native lane); at most once per response, before the first audio delta. | +| `response.audio.delta` | 2 | One ordered audio chunk. OpenAI name; adds `format`, `sample_rate_hz`, `metadata{epoch, model_speak, end_of_turn, audio_text_marks, playback}`. | +| `response.audio_transcript.delta` | 1 | Transcript text paired one-to-one with each audio delta. | +| `response.output_text.delta` | 1 | Text delta for text-modality responses. | +| `response.output_text.done` | 1 | Final text of a text-modality response. | +| `response.audio.done` | 1 | Audio stream closed for the response. | +| `response.audio_transcript.done` | 1 | Full transcript (concatenation of the deltas). | +| `response.content_part.done` | 1 | Content part finalized with its transcript/text. | +| `response.output_item.done` | 1 | Assistant item finalized. | +| `response.done` | 2 | Terminal event; `status` ∈ `completed` \| `cancelled` \| `failed`. OpenAI name; vLLM-Omni `status_details.reason` values and raw duplex event under `metadata`. | +| `rate_limits.updated` | 1 | Compatibility event after every `response.done`; always an empty list. | +| `response.listen` | 3 | Model chose to keep listening at a unit boundary or after a silence commit. Carries `response_id` when the decision terminates a precreated response (a `response.done` with the same id follows); otherwise no `response.created` precedes it. | +| `response.function_call_arguments.delta` | 1 | Tool-call arguments delta (Nemotron VoiceChat). | +| `response.function_call_arguments.done` | 1 | Tool-call arguments complete. | +| `duplex.function_call.done` | 3 | Raw fallback when a model tool call is malformed. | +| `output_audio_buffer.cleared` | 1 | Reply to `output_audio_buffer.clear`. | +| `playback.acknowledged` | 3 | Reply to `playback.ack`; returns the playback ledger and `history_committed`. | +| `overlap.decision` | 3 | How an append that overlapped an active response was handled (`drop` \| `listen` \| `barge_in`). | +| `runtime.control` | 3 | Diagnostic: an engine control signal was partially unsupported. | +| `error` | 2 | Error envelope. OpenAI shape and classes; vLLM-Omni error codes. | + +### Message examples + +Field values are illustrative; base64 payloads are shortened to `"…"`. +`C→S` = client sends, `S→C` = server sends. Every `S→C` event on the wire +additionally carries `"server_event_seq": ` (omitted below for brevity). + +#### Session lifecycle + +`C→S session.update` (first one = open handshake) + +```json +{ + "type": "session.update", + "event_id": "evt_0001", + "session": { + "model": "openbmb/MiniCPM-o-4_5", + "session_id": "sess_demo_01", + "modalities": ["audio", "text"], + "instructions": "You are a concise voice assistant.", + "input_audio_format": "pcm16", + "output_audio_format": "pcm16", + "ref_audio": "data:audio/wav;base64,…", + "turn_detection": null, + "overlap_policy": "listen_only", + "playback_commit_policy": "ack_only", + "temperature": 0.7, + "idle_timeout_s": 300, + "extra_body": { + "auto_response": true, + "native_duplex": true, + "force_listen_count": 0 + } + } +} +``` + +`S→C session.created` + +```json +{ + "type": "session.created", + "incarnation": 0, + "attachment_generation": 1, + "resume_token": "rt_3f9c…", + "session": { + "object": "realtime.session", + "type": "realtime", + "id": "sess_demo_01", + "model": "openbmb/MiniCPM-o-4_5", + "state": "open", + "turn_state": "idle", + "epoch": 0, + "turn_id": 0, + "active_request_id": null, + "active_response_id": null, + "active_response_turn_id": null, + "modalities": ["audio", "text"], + "output_modalities": ["audio", "text"], + "instructions": "You are a concise voice assistant.", + "voice": null, + "response_format": "pcm16", + "input_audio_format": "pcm16", + "output_audio_format": "pcm16", + "audio": { + "input": {"format": {"type": "audio/pcm", "rate": 16000}, "sample_rate_hz": 16000}, + "output": {"format": {"type": "audio/pcm", "rate": 24000}} + }, + "temperature": 0.7, + "max_tokens": null, + "speed": null, + "idle_timeout_s": 300.0, + "overlap_policy": "listen_only", + "overlap_short_ack_ms": 700, + "overlap_barge_in_ms": 1200, + "overlap_silence_rms": 0.003, + "playback_commit_policy": "ack_only", + "turn_detection": null, + "input_audio_transcription": null, + "tracing": null, + "playback": {"generated_ms": 0, "sent_ms": 0, "played_ms": 0, "committed_ms": 0}, + "capabilities": { + "implementation_level": "model_native_duplex", + "supports_input_append": true, + "supports_barge_in": true, + "supports_session_resume": true, + "supports_session_lease": true, + "supports_multi_session": true, + "chunk_period_ms": 1000, + "input_modes": ["append_audio_chunk"], + "signal_sources": ["model_native", "client_event", "server_policy"], + "target_barge_in_latency_ms": null + } + } +} +``` + +`S→C session.updated` (same `session` object; follows the first +`session.created` and every later `session.update`) + +```json +{"type": "session.updated", "session": {"object": "realtime.session", "id": "sess_demo_01", "…": "…"}} +``` + +`C→S session.heartbeat` / `S→C session.heartbeat_ack` + +```json +{"type": "session.heartbeat"} +``` + +```json +{"type": "session.heartbeat_ack", "session_id": "sess_demo_01"} +``` + +`C→S session.event_ack` + +```json +{"type": "session.event_ack", "server_event_seq": 48} +``` + +`C→S session.close` / `S→C session.closed` + +```json +{"type": "session.close"} +``` + +```json +{"type": "session.closed", "session_id": "sess_demo_01", "reason": "session_close"} +``` + +`C→S session.resume` (new socket, `?duplex=1&autostart=0`) + +```json +{ + "type": "session.resume", + "session_id": "sess_demo_01", + "incarnation": 0, + "resume_token": "rt_3f9c…", + "last_received_server_event_seq": 40 +} +``` + +`S→C session.resumed` (activation; the token is rotated, then journaled +events with `server_event_seq > 40` are replayed in order) + +```json +{ + "type": "session.resumed", + "session_id": "sess_demo_01", + "incarnation": 0, + "attachment_generation": 2, + "resume_token": "rt_a71e…" +} +``` + +`S→C session.replaced` (sent to the superseded socket, which is then closed) + +```json +{"type": "session.replaced", "session_id": "sess_demo_01", "attachment_generation": 1} +``` + +`S→C session.expired` + +```json +{"type": "session.expired", "session_id": "sess_demo_01", "incarnation": 0, "reason": "disconnect_grace_expired"} +``` + +`S→C session.resync_required` + +```json +{"type": "session.resync_required", "session_id": "sess_demo_01", "reason": "journal_gap"} +``` + +#### Input + +`C→S input_audio_buffer.append` (200 ms of 16 kHz PCM16 = 6400 bytes) + +```json +{ + "type": "input_audio_buffer.append", + "event_id": "evt_0042", + "audio": "…", + "input_audio_format": "pcm16", + "sample_rate_hz": 16000, + "duration_ms": 200, + "audio_end_ms": 1200, + "is_speech": true +} +``` + +with camera frames (omni video; rides the append that closes a 1 s model +unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix) + +```json +{ + "type": "input_audio_buffer.append", + "audio": "…", + "input_audio_format": "pcm16", + "sample_rate_hz": 16000, + "duration_ms": 200, + "audio_end_ms": 1200, + "video_frames": ["/9j/4AAQSkZJRg…", "/9j/4AAQSkZJRg…"] +} +``` + +`S→C input_audio_buffer.speech_started` / `speech_stopped` + +```json +{"type": "input_audio_buffer.speech_started", "audio_start_ms": 1040, "item_id": "item_9b2d…"} +``` + +```json +{"type": "input_audio_buffer.speech_stopped", "audio_end_ms": 2860, "item_id": "item_9b2d…"} +``` + +`C→S input_audio_buffer.commit` + +```json +{"type": "input_audio_buffer.commit", "event_id": "evt_0060", "final": true} +``` + +alias form accepted by the runner (`input.commit`) + +```json +{"type": "input.commit", "final": true} +``` + +silence/noise declaration (no runtime append; answered with `response.listen`) + +```json +{"type": "input_audio_buffer.commit", "final": true, "is_speech": false} +``` + +`S→C` commit fan-out (in this order) + +```json +{ + "type": "conversation.item.added", + "previous_item_id": null, + "item": { + "id": "item_9b2d…", + "object": "realtime.item", + "type": "message", + "role": "user", + "status": "completed", + "content": [{"type": "input_audio", "transcript": "what is the weather today"}] + } +} +``` + +```json +{"type": "conversation.item.created", "previous_item_id": null, "item": {"id": "item_9b2d…", "…": "…"}} +``` + +```json +{ + "type": "input_audio_buffer.committed", + "previous_item_id": null, + "item_id": "item_9b2d…", + "event": { + "type": "input.committed", + "session_id": "sess_demo_01", + "turn_id": 1, + "epoch": 0, + "history_len": 1, + "message": {"role": "user", "content": [{"type": "audio_url", "audio_url": {"url": "native-duplex:input-audio"}}]}, + "realtime_item_id": "item_9b2d…" + } +} +``` + +```json +{ + "type": "conversation.item.input_audio_transcription.completed", + "item_id": "item_9b2d…", + "content_index": 0, + "transcript": "what is the weather today" +} +``` + +```json +{"type": "conversation.item.done", "previous_item_id": null, "item": {"id": "item_9b2d…", "status": "completed", "…": "…"}} +``` + +deferred commit during an active response (`event.response_create_deferred`) + +```json +{ + "type": "input_audio_buffer.committed", + "previous_item_id": "item_9b2d…", + "item_id": "item_c044…", + "event": {"type": "input.committed", "session_id": "sess_demo_01", "turn_id": 2, "epoch": 0, "history_len": 3, "response_create_deferred": true, "message": {"…": "…"}} +} +``` + +`C→S input_audio_buffer.clear` / `S→C input_audio_buffer.cleared` + +```json +{"type": "input_audio_buffer.clear"} +``` + +```json +{"type": "input_audio_buffer.cleared"} +``` + +`C→S input.cancel` (advances `epoch`; also answered with `input_audio_buffer.cleared`) + +```json +{"type": "input.cancel"} +``` + +`C→S input.text.append` + +```json +{"type": "input.text.append", "text": "and tomorrow?"} +``` + +`C→S conversation.item.create` (text turn, or a tool result) + +```json +{ + "type": "conversation.item.create", + "previous_item_id": null, + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Summarise what I said."}] + } +} +``` + +```json +{ + "type": "conversation.item.create", + "item": {"type": "function_call_output", "call_id": "call_7d1f", "output": "{\"temperature_c\": 24}"} +} +``` + +`C→S conversation.item.delete` / `S→C conversation.item.deleted` + +```json +{"type": "conversation.item.delete", "item_id": "item_9b2d…"} +``` + +```json +{"type": "conversation.item.deleted", "item_id": "item_9b2d…", "event": {"type": "conversation.item.deleted", "item_id": "item_9b2d…"}} +``` + +`C→S conversation.item.retrieve` / `S→C conversation.item.retrieved` + +```json +{"type": "conversation.item.retrieve", "item_id": "item_resp_01"} +``` + +```json +{"type": "conversation.item.retrieved", "item": {"id": "item_resp_01", "object": "realtime.item", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_audio", "transcript": "It is sunny, 24 degrees."}]}} +``` + +`C→S conversation.item.truncate` / `S→C conversation.item.truncated` + +```json +{"type": "conversation.item.truncate", "item_id": "item_resp_01", "content_index": 0, "audio_end_ms": 1850} +``` + +```json +{"type": "conversation.item.truncated", "item_id": "item_resp_01", "content_index": 0, "audio_end_ms": 1850, "event": {"…": "…"}} +``` + +#### Response lifecycle + +`C→S response.create` + +```json +{ + "type": "response.create", + "event_id": "evt_0070", + "response": { + "modalities": ["audio", "text"], + "instructions": "Answer in one sentence.", + "output_audio_format": "pcm16", + "temperature": 0.6, + "max_output_tokens": 256, + "metadata": {"trace": "turn-3"} + } +} +``` + +`S→C response.created` + +```json +{ + "type": "response.created", + "response_id": "resp_01", + "response": { + "id": "resp_01", + "object": "realtime.response", + "status": "in_progress", + "status_details": null, + "output": [], + "modalities": ["audio", "text"], + "metadata": {"duplex_event": {"type": "response.created", "session_id": "sess_demo_01", "response_id": "resp_01", "epoch": 0, "turn_id": 1}} + } +} +``` + +`S→C conversation.item.added` / `conversation.item.created` (assistant item, `status: in_progress`) + +```json +{ + "type": "conversation.item.added", + "previous_item_id": "item_9b2d…", + "item": {"id": "item_resp_01", "object": "realtime.item", "type": "message", "role": "assistant", "status": "in_progress", "content": []} +} +``` + +`S→C response.output_item.added` + +```json +{"type": "response.output_item.added", "response_id": "resp_01", "output_index": 0, "item": {"id": "item_resp_01", "object": "realtime.item", "type": "message", "role": "assistant", "status": "in_progress", "content": []}} +``` + +`S→C response.content_part.added` + +```json +{"type": "response.content_part.added", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0, "part": {"type": "audio", "transcript": ""}} +``` + +`S→C response.speak` + +```json +{ + "type": "response.speak", + "response_id": "resp_01", + "item_id": "item_resp_01", + "output_index": 0, + "content_index": 0, + "metadata": {"session_id": "sess_demo_01", "epoch": 0, "model_speak": true} +} +``` + +`S→C response.audio.delta` (one ~1 s unit, 24 kHz PCM16) + +```json +{ + "type": "response.audio.delta", + "response_id": "resp_01", + "item_id": "item_resp_01", + "output_index": 0, + "content_index": 0, + "delta": "…", + "format": "pcm16", + "sample_rate_hz": 24000, + "metadata": { + "session_id": "sess_demo_01", + "epoch": 0, + "model_speak": true, + "end_of_turn": false, + "audio_duration_ms": 1000, + "audio_text_marks": [{"text_chars": 11, "audio_end_ms": 1000}], + "playback": {"generated_ms": 1000, "sent_ms": 1000, "played_ms": 0, "committed_ms": 0} + } +} +``` + +`S→C response.audio_transcript.delta` (exactly one per audio delta) + +```json +{"type": "response.audio_transcript.delta", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0, "delta": "It is sunny"} +``` + +`S→C response.output_text.delta` / `response.output_text.done` (text modality) + +```json +{"type": "response.output_text.delta", "response_id": "resp_02", "item_id": "item_resp_02", "output_index": 0, "content_index": 0, "delta": "Tomorrow looks"} +``` + +```json +{"type": "response.output_text.done", "response_id": "resp_02", "item_id": "item_resp_02", "output_index": 0, "content_index": 0, "text": "Tomorrow looks cloudy."} +``` + +`S→C response.audio.done` / `response.audio_transcript.done` + +```json +{"type": "response.audio.done", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0} +``` + +```json +{"type": "response.audio_transcript.done", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0, "transcript": "It is sunny, 24 degrees."} +``` + +`S→C response.content_part.done` + +```json +{"type": "response.content_part.done", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0, "part": {"type": "audio", "transcript": "It is sunny, 24 degrees."}} +``` + +`S→C response.output_item.done` / `conversation.item.done` + +```json +{"type": "response.output_item.done", "response_id": "resp_01", "output_index": 0, "item": {"id": "item_resp_01", "object": "realtime.item", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_audio", "transcript": "It is sunny, 24 degrees."}]}} +``` + +```json +{"type": "conversation.item.done", "previous_item_id": "item_9b2d…", "item": {"id": "item_resp_01", "status": "completed", "…": "…"}} +``` + +`S→C response.done` (completed) + +```json +{ + "type": "response.done", + "response_id": "resp_01", + "response": { + "id": "resp_01", + "object": "realtime.response", + "status": "completed", + "status_details": null, + "output": [{"id": "item_resp_01", "object": "realtime.item", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_audio", "transcript": "It is sunny, 24 degrees."}]}], + "metadata": {"type": "response.done", "session_id": "sess_demo_01", "response_id": "resp_01", "epoch": 0, "committed": true, "playback": {"generated_ms": 2400, "sent_ms": 2400, "played_ms": 2400, "committed_ms": 2400}} + } +} +``` + +`S→C response.done` (cancelled by barge-in) + +```json +{ + "type": "response.done", + "response_id": "resp_01", + "response": {"id": "resp_01", "object": "realtime.response", "status": "cancelled", "status_details": {"type": "cancelled", "reason": "barge_in"}, "output": [{"id": "item_resp_01", "status": "cancelled", "…": "…"}], "metadata": {"…": "…"}} +} +``` + +`S→C rate_limits.updated` + +```json +{"type": "rate_limits.updated", "rate_limits": []} +``` + +`S→C response.listen` + +```json +{ + "type": "response.listen", + "session_id": "sess_demo_01", + "epoch": 0, + "response": {"object": "realtime.response", "status": "listening", "metadata": {"type": "response.listen", "session_id": "sess_demo_01", "epoch": 0, "reason": "model_listen"}} +} +``` + +`C→S response.cancel` + +```json +{"type": "response.cancel", "response_id": "resp_01"} +``` + +`C→S output_audio_buffer.clear` / `S→C output_audio_buffer.cleared` + +```json +{"type": "output_audio_buffer.clear"} +``` + +```json +{"type": "output_audio_buffer.cleared", "response_id": "resp_01"} +``` + +`C→S barge_in` / `turn.signal` + +```json +{"type": "barge_in"} +``` + +```json +{"type": "turn.signal", "event": "barge_in", "payload": {"source": "client_vad"}} +``` + +function-call response (Nemotron VoiceChat tools) + +```json +{"type": "response.output_item.added", "response_id": "resp_fc_01", "output_index": 0, "item": {"id": "item_fc_01", "object": "realtime.item", "type": "function_call", "status": "completed", "name": "get_weather", "call_id": "call_7d1f", "arguments": "{\"city\":\"Hong Kong\"}"}} +``` + +```json +{"type": "response.function_call_arguments.delta", "response_id": "resp_fc_01", "item_id": "item_fc_01", "output_index": 0, "call_id": "call_7d1f", "delta": "{\"city\":\"Hong Kong\"}"} +``` + +```json +{"type": "response.function_call_arguments.done", "response_id": "resp_fc_01", "item_id": "item_fc_01", "output_index": 0, "call_id": "call_7d1f", "arguments": "{\"city\":\"Hong Kong\"}"} +``` + +raw fallback when a model tool call is malformed (`call_id`/`name` missing) + +```json +{"type": "duplex.function_call.done", "event": {"type": "function_call.done", "session_id": "sess_demo_01", "epoch": 0, "call_id": null, "name": null, "arguments": "{"}} +``` + +#### Playback + +`C→S playback.ack` / `S→C playback.acknowledged` + +```json +{"type": "playback.ack", "response_id": "resp_01", "item_id": "item_resp_01", "played_ms": 1000, "committed_ms": 1000} +``` + +```json +{ + "type": "playback.acknowledged", + "event": { + "type": "playback.acknowledged", + "session_id": "sess_demo_01", + "epoch": 0, + "item_id": "item_resp_01", + "played_ms": 1000, + "committed_ms": 1000, + "truncate": false, + "playback": {"generated_ms": 2400, "sent_ms": 2400, "played_ms": 1000, "committed_ms": 1000}, + "history_committed": false + } +} +``` + +truncating ack after an interruption (what was actually heard) + +```json +{"type": "playback.ack", "response_id": "resp_01", "played_ms": 1850, "committed_ms": 1850, "truncate": true} +``` + +#### Overlap and barge-in + +`S→C overlap.decision` + +```json +{ + "type": "overlap.decision", + "session_id": "sess_demo_01", + "epoch": 0, + "policy": "listen_only", + "action": "listen", + "reason": "long_overlap_speech", + "overlap_ms": 1400, + "buffer_audio": true, + "defer_runtime_append": true, + "force_listen": true +} +``` + +```json +{"type": "overlap.decision", "session_id": "sess_demo_01", "epoch": 0, "policy": "barge_in_on_speech", "action": "barge_in", "reason": "server_vad_utterance_active"} +``` + +server-VAD session (`session.update` fragment) and the annotated append the +translator forwards internally (shown for reference; clients never see it) + +```json +{"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "threshold": 0.5, "prefix_padding_ms": 300, "silence_duration_ms": 500}, "overlap_policy": "barge_in_on_speech"}} +``` + +```json +{"type": "input_audio_buffer.append", "audio": "…", "format": "pcm16", "sample_rate_hz": 16000, "is_speech": true, "force_listen": true, "vad": {"backend": "silero", "is_speech": true, "speech_active": true, "speech_started": true, "speech_stopped": false, "speech_probability": 0.93}} +``` + +`S→C runtime.control` (only when an engine signal was partially unsupported) + +```json +{"type": "runtime.control", "session_id": "sess_demo_01", "epoch": 1, "result": {"…": "redacted engine control result"}} +``` + +#### Errors + +`S→C error` — Realtime shape + +```json +{ + "type": "error", + "error": { + "type": "invalid_request_error", + "code": "input_audio_buffer_empty", + "message": "input_audio_buffer.commit requires a non-empty input audio buffer.", + "event_id": "evt_0060" + } +} +``` + +```json +{"type": "error", "error": {"type": "invalid_request_error", "code": "unsupported_turn_detection", "message": "turn_detection.interrupt_response=false is unsupported; use turn_detection=null for model-owned listen/speak", "event_id": "evt_0001", "param": "turn_detection"}} +``` + +```json +{"type": "error", "error": {"type": "server_error", "code": "runtime_append_failed", "message": "stale epoch: expected 1, got 0"}} +``` + +```json +{"type": "error", "error": {"type": "rate_limit_error", "code": "duplex_session_busy", "message": "duplex session capacity reached (max_sessions=4)"}} +``` + +Handshake-stage errors (before the Realtime projector is bound) use the +flat native shape: + +```json +{"type": "error", "error": "Unknown or expired duplex session: sess_demo_01", "code": "session_resume_expired"} +``` + +#### Coverage + +The examples above cover every event type the Realtime route emits or +accepts (verified by diffing the `type` literals in +`vllm_omni/entrypoints/duplex/` against this document). Not shown, because +they never reach a `/v1/realtime?duplex=1` client unprojected: the native +`/v1/duplex` dialect (`session.create` / `open_session` / `session.config`, +`input.committed`, `input.cancelled`, `audio.cancelled`, +`response.output_audio.delta`, `response.text.delta`, `response.message`, +`function_call.done`) — their Realtime projections are the examples above +(see the name map at the end of this section) — and the runner-internal markers `__timeout__`, +`__disconnect__`, `__replaced_attachment__`. Input aliases (`push_chunk`, +`input.audio.append`, `input_text.append`, `push_text`, `signal_turn`, +`audio.playback_ack`, `close_session`, `close`, `session_close`) share the +payload of their canonical event (see the alias table in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md)). + +#### Native to Realtime name map + +The native `/v1/duplex` dialect is what the session runner produces; the +Realtime projector renames and fans out these events before they reach a +`/v1/realtime?duplex=1` client. + +| Native / internal event | Realtime projection | +| --- | --- | +| `session.create` (`open_session`, `session.config`) | first `session.update` | +| `input.committed` | `conversation.item.added` / `conversation.item.created`, `input_audio_buffer.committed`, `conversation.item.input_audio_transcription.completed`, `conversation.item.done` | +| `input.cancelled` | `input_audio_buffer.cleared` | +| `response.output_audio.delta` (with `audio_transcript`) | optional `response.speak`, then `response.audio.delta` and `response.audio_transcript.delta` | +| `response.output_audio.done` / `response.output_text.done` | `response.audio.done` / `response.output_text.done` | +| `response.text.delta` | `response.output_text.delta` | +| `response.message` (chat-fallback raw chunk) | passed through | +| `audio.cancelled` | optional `output_audio_buffer.cleared`, then the cancelled terminal events and `response.done` with `status: "cancelled"` | +| `function_call.done` | `response.created`, `response.output_item.added` (`function_call` item), `response.function_call_arguments.delta` / `.done`, `response.output_item.done`, `response.done` | +| `playback.acknowledged` | `playback.acknowledged` (wrapped verbatim under `event`) | +| `runtime.control`, `session.resumed`, `session.replaced`, `session.expired`, `session.resync_required`, `session.heartbeat_ack` | passed through unchanged | + +## Known Limitations + +- Several surfaces are capability-gated per model (see *Capability + negotiation by model* above): PersonaPlex does not support session resume, + barge-in, or audio truncation; Nemotron VoiceChat does not support barge-in + or audio truncation; camera frames are consumed only by MiniCPM-o 4.5; tool + calls are produced only by Nemotron VoiceChat. +- `turn_detection` supports only `server_vad` with `interrupt_response=true`; + `semantic_vad`, `interrupt_response=false`, and `create_response` are not + supported. Server VAD requires a model-native session. +- `input_audio_transcription` and `input_audio_noise_reduction` are accepted + and echoed but no separate transcription or noise-reduction stage runs; + transcripts come from the model. +- `rate_limits.updated` is emitted for compatibility only and always carries + an empty list. +- Session capacity is bounded by `duplex_session.max_sessions` in the deploy + configuration; admission beyond it fails with `duplex_session_busy` or + `resource_exhausted`. +- The native `/v1/duplex` route speaks the internal dialect and does not + support `session.resume`; use `/v1/realtime?duplex=1` for the contract on + this page. diff --git a/docs/serving/standalone_servers.md b/docs/serving/standalone_servers.md index 66e0c1e08..6eb464833 100644 --- a/docs/serving/standalone_servers.md +++ b/docs/serving/standalone_servers.md @@ -1,61 +1,19 @@ # Standalone Experimental Servers Most vLLM-Omni endpoints run on the unified server started with -`vllm serve --omni`. The experimental full-duplex package also contains -two separate user-facing server processes. Their routes are not added to the -unified API server. +`vllm serve --omni`. The experimental package also contains one +separate user-facing server process; its routes are not added to the unified +API server. | Server | Transport | Role | -|--------|-----------|------| -| PersonaPlex compatibility server | WebSocket | Hosts the official PersonaPlex browser protocol and a raw-PCM alternative | +| --- | --- | --- | | JoyVL interaction server | HTTP | Adds state, memory, proactive decisions, and delegation in front of a separate OpenAI-compatible model backend | -## PersonaPlex Compatibility Server - -Use this server when a client requires the official Moshi/PersonaPlex wire -protocol or the bundled browser UI. For the engine-owned unified duplex path, -use [`WS /v1/realtime?duplex=1`](full_duplex_api.md) instead. - -Start it with: - -```bash -HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m \ - vllm_omni.experimental.fullduplex.personaplex.serving.server \ - --port 8091 -``` - -| Route | Purpose | -|-------|---------| -| `GET /` | Official PersonaPlex browser client, when its assets are available | -| `GET /health` | Readiness check | -| `WS /api/chat` | Moshi-compatible tagged binary protocol with Opus audio | -| `WS /v1/audio/duplex` | Simpler float32 raw-PCM protocol | - -### `/api/chat` Protocol - -The first byte of every binary frame is a message tag: - -| Direction | Tag | Payload | -|-----------|-----|---------| -| Server to client | `0x00` | Ready handshake | -| Client to server | `0x01` | Opus microphone audio at 24 kHz | -| Server to client | `0x01` | Opus response audio | -| Server to client | `0x02` | UTF-8 inner-monologue text | - -The optional `text_prompt` and `voice_prompt` query parameters select a persona -and voice. The default server admits one conversation; use `--batch-size N` to -configure multiple lockstep slots. Connections beyond capacity close with -WebSocket code 1013. - -### `/v1/audio/duplex` Protocol - -Send `{"type":"open","persona":"...","voice":"..."}` as a JSON text frame, -then exchange binary float32 PCM frames. The server returns generated float32 -audio as binary frames and text as `{"type":"text","text":"..."}`. Send -`{"type":"close"}` and wait for `{"type":"done"}` to flush and finish. - -See the [PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex) -for the browser and headless clients. +PersonaPlex no longer ships a standalone compatibility server: it is served +through the unified duplex path (`vllm_omni/deploy/personaplex.yaml`) over +[`WS /v1/realtime?duplex=1`](full_duplex_api.md); see the +[PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex) +for its clients. ## JoyVL Interaction Server @@ -80,7 +38,7 @@ python -m vllm_omni.experimental.fullduplex.joyvl.serving.server \ ``` | Method and route | Purpose | -|------------------|---------| +| ------------------ | --------- | | `GET /health` | Readiness check | | `GET /v1/models` | Reports the configured interaction model | | `POST /v1/chat/completions` | Processes one frame or interaction tick | diff --git a/docs/usage/faq.md b/docs/usage/faq.md index 87b63998e..7e144cea8 100644 --- a/docs/usage/faq.md +++ b/docs/usage/faq.md @@ -18,4 +18,4 @@ A: AWQ is available through the inherited vLLM quantization registry, and vLLM-O > Q: Does vLLM-Omni support multimodal streaming input and output? -A: Yes, for specific models and serving paths. Current capabilities include [streaming video input](../serving/video_stream_api.md), [streaming diffusion output](../user_guide/diffusion/execution_modes.md#streaming-output), and experimental full-duplex real-time audio described in the [project overview](../README.md). Check the model and API documentation because input and output support is model-specific. +A: Yes, for specific models and serving paths. Current capabilities include [streaming video input](../serving/video_stream_api.md), [streaming diffusion output](../user_guide/diffusion/execution_modes.md#streaming-output), and [full-duplex realtime audio](../serving/realtime_duplex_api.md). Check the model and API documentation because input and output support is model-specific. diff --git a/examples/offline_inference/personaplex/personaplex_offline.py b/examples/offline_inference/personaplex/personaplex_offline.py deleted file mode 100644 index ea011b8a9..000000000 --- a/examples/offline_inference/personaplex/personaplex_offline.py +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Offline PersonaPlex inference through the vllm-omni fullduplex backend. - -This is the runnable end-to-end proof for the integration: it drives a user WAV -through :class:`PersonaPlexEngine` + :class:`PersonaPlexSession` (our wrappers -around the Moshi/Mimi lockstep loop) and writes the agent's audio + inner-monologue -text. It mirrors ``python -m moshi.offline`` but goes through our ``FrameStepper`` -seam, so a green run validates that the session/adapter plumbing is faithful. - -Prereqs: HF access to ``nvidia/personaplex-7b-v1`` (``export HF_TOKEN=...``). The -stack is moshi-free (native temporal + depformer + streaming Mimi via -``transformers``); decoding is greedy. - -Example: - python examples/offline_inference/personaplex/personaplex_offline.py \\ - --input-wav input_assistant.wav --output-wav out.wav \\ - --voice-prompt NATF2.pt --persona "You are a wise and friendly teacher." -""" - -from __future__ import annotations - -import argparse -import json - -import numpy as np - -from vllm_omni.experimental.fullduplex.personaplex import ( - PersonaPlexConfig, - PersonaPlexEngine, - PersonaPlexSession, -) -from vllm_omni.experimental.fullduplex.personaplex.config import DEFAULT_PERSONA - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--input-wav", required=True, help="User-side input WAV") - parser.add_argument("--output-wav", required=True, help="Agent audio output WAV") - parser.add_argument("--output-text", default=None, help="Optional JSON for agent text") - parser.add_argument("--voice-prompt", default="NATF2.pt", help="Voice clone (bundled .pt or .wav)") - parser.add_argument("--persona", default=DEFAULT_PERSONA, help="System role text") - parser.add_argument("--hf-repo", default="nvidia/personaplex-7b-v1") - parser.add_argument("--device", default="cuda") - parser.add_argument("--cpu-offload", action="store_true") - args = parser.parse_args() - - import sphn # local import: only the runnable driver needs the audio I/O dep - - config = PersonaPlexConfig( - hf_repo=args.hf_repo, - voice_prompt=args.voice_prompt, - persona=args.persona, - device=args.device, - cpu_offload=args.cpu_offload, - ) - - engine = PersonaPlexEngine(config).load() - session = PersonaPlexSession(engine, config) - session.open() - - # Load exactly like the reference (moshi.lm.load_audio): native read then - # explicit resample, so greedy runs stay bit-faithful to `python -m moshi.offline`. - user_pcm, src_sr = sphn.read(args.input_wav) - user_pcm = sphn.resample(user_pcm, src_sample_rate=src_sr, dst_sample_rate=config.sample_rate) - audio, text = session.run(np.asarray(user_pcm[0], dtype=np.float32)) - - sphn.write_wav(args.output_wav, audio, config.sample_rate) - print(f"wrote {audio.shape[0] / config.sample_rate:.2f}s of agent audio to {args.output_wav}") - print(f"agent text: {text!r}") - if args.output_text: - with open(args.output_text, "w") as f: - json.dump({"text": text}, f, ensure_ascii=False) - - -if __name__ == "__main__": - main() diff --git a/examples/online_serving/barge_in_client.py b/examples/online_serving/barge_in_client.py new file mode 100644 index 000000000..1ac627d35 --- /dev/null +++ b/examples/online_serving/barge_in_client.py @@ -0,0 +1,272 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Full-duplex use case: talk over the assistant (barge-in). + +Scenario (see ``barge_in_client_flow.md`` for the diagram) +--------------------------------------------------------------- +1. The user asks a question (``--question-wav``); the model answers in speech. +2. Mid-answer, the user starts talking again (``--interrupt-wav``) — a + follow-up or correction — while assistant audio is still streaming. +3. The model handles the overlap: depending on the session overlap policy and + speech length, the in-flight response is barged in (cancelled) or the + commit is deferred to the turn end, and a new response answers the + interruption. Either way the user never waits for the first answer. + +The demo is model-neutral: it drives any duplex model through the public +:class:`vllm_omni.clients.duplex.DuplexClient` over +``/v1/realtime?duplex=1``. Model-specific session shape comes from the +per-model presets under ``vllm_omni.clients.`` selected with +``--preset`` (default ``minicpmo_4_5``, whose model decides listen/speak on +its own — no VAD). + +Run the server first (for MiniCPM-o 4.5: ``bash minicpmo/barge_in_serve.sh``, which +starts the ``vllm_omni/deploy/minicpmo_4_5.yaml`` profile with ``session_mode: duplex``), +then: + + python barge_in_client.py \ + --url ws://127.0.0.1:8099/v1/realtime \ + --model openbmb/MiniCPM-o-4_5 \ + --ref-audio /path/to/reference_voice.wav \ + --question-wav question_16k.wav \ + --interrupt-wav follow_up_16k.wav \ + --output-dir ./duplex_out + +Input WAVs must be mono 16 kHz PCM16; when a preset's session takes a +different capture format (the PersonaPlex preset streams 24 kHz float32), +the audio is converted before streaming. ``--ref-audio`` is required by the +MiniCPM-o preset (without it the session is rejected with +``ref_audio_required``). Outputs land in ``--output-dir`` as one WAV per +response — ``response_1_cancelled.wav`` for a barged-in answer, +``response_1_completed.wav`` for one that finished — plus ``summary.json``. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from vllm_omni.clients.duplex import ( # noqa: E402 + AudioFormat, + DuplexClient, + EventCollector, + SessionConfig, + acknowledge_collected_playback, + audio_data_url, + read_pcm16_wav, + wait_for_condition, + write_pcm16_wav, +) + +_TRANSCRIPT_DELTA_EVENTS = { + "response.audio_transcript.delta", + "response.output_text.delta", + "response.text.delta", +} + +# Input WAVs are mono 16 kHz PCM16; each preset's session may negotiate a +# different capture format (PersonaPlex takes 24 kHz float32) and a different +# model-unit-aligned chunk size (PersonaPlex frames are exactly 80 ms). +_PRESET_DEFAULT_CHUNK_MS = {"minicpmo_4_5": 100, "personaplex": 80, "none": 100} + + +def _convert_to_session_format(pcm16_16k: bytes, target: AudioFormat) -> bytes: + """Convert mono 16 kHz PCM16 to the session's input format.""" + if target.encoding == "pcm16" and target.sample_rate_hz == 16_000: + return pcm16_16k + import numpy as np + + samples = np.frombuffer(pcm16_16k, dtype=np.int16).astype(np.float32) / 32768.0 + if target.sample_rate_hz != 16_000 and len(samples) > 1: + target_len = round(len(samples) * target.sample_rate_hz / 16_000) + positions = np.linspace(0.0, len(samples) - 1, num=target_len) + samples = np.interp(positions, np.arange(len(samples)), samples).astype(np.float32) + if target.encoding == "pcm_f32le": + return samples.tobytes() + if target.encoding == "pcm16": + return (np.clip(samples, -1.0, 1.0) * 32767.0).astype(np.int16).tobytes() + raise ValueError(f"Unsupported session input encoding: {target.encoding!r}") + + +def _session_config(preset: str, *, ref_audio_path: str | None) -> SessionConfig: + ref_audio = audio_data_url(Path(ref_audio_path).expanduser()) if ref_audio_path else None + if preset == "minicpmo_4_5": + from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config + + return create_duplex_session_config(ref_audio=ref_audio) + if preset == "personaplex": + from vllm_omni.clients.personaplex import create_duplex_session_config + + return create_duplex_session_config() + return SessionConfig(ref_audio=ref_audio) + + +def _fold_responses(collector: EventCollector) -> dict[str, dict[str, object]]: + """Fold the event stream into per-response audio, text, and status.""" + out: dict[str, dict[str, object]] = {} + for response_id in collector.response_ids: + out[response_id] = { + "audio": collector.audio_bytes(response_id), + "text": "", + "status": "in_progress", + "sample_rate_hz": collector.output_sample_rate_hz, + } + + for event in collector.events: + etype = event.get("type") + response_id = collector.response_id(event) + item = out.get(response_id) if isinstance(response_id, str) else None + if etype in _TRANSCRIPT_DELTA_EVENTS and item is not None: + delta = event.get("delta") + if isinstance(delta, str): + item["text"] += delta + elif etype == "response.done" and item is not None: + response = event.get("response") + status = response.get("status") if isinstance(response, dict) else None + if item["status"] == "in_progress": + item["status"] = "cancelled" if status == "cancelled" else "completed" + elif etype == "audio.cancelled": + targets = [item] if item is not None else list(out.values()) + for target in targets: + if target["status"] == "in_progress": + target["status"] = "cancelled" + return out + + +async def run(args: argparse.Namespace) -> int: + config = _session_config(args.preset, ref_audio_path=args.ref_audio) + question = _convert_to_session_format(read_pcm16_wav(Path(args.question_wav)), config.input_audio) + interrupt = _convert_to_session_format(read_pcm16_wav(Path(args.interrupt_wav)), config.input_audio) + chunk_ms = args.chunk_ms if args.chunk_ms is not None else _PRESET_DEFAULT_CHUNK_MS.get(args.preset, 100) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + collector = EventCollector() + client = DuplexClient( + args.url, + model=args.model, + config=config, + reconnect=None, + heartbeat_interval_s=None, + handshake_timeout_s=args.timeout_s, + ) + # GREET: open the voice session (handshake happens on enter). + async with client: + consume_task = asyncio.create_task(collector.consume(client)) + + # ASK: stream the question at realtime pace, then commit the turn. + # The model decides on its own to answer (no VAD). + print("[1] streaming question ...", file=sys.stderr) + await client.stream_pcm(question, chunk_ms=chunk_ms) + await client.commit(create_response=False) + await wait_for_condition( + lambda: collector.count("response.created") > 0, + timeout_s=args.timeout_s, + label="first response.created", + ) + print("[1] assistant is answering", file=sys.stderr) + + # Let the answer play for a moment, as a listening user would. + await asyncio.sleep(args.listen_s) + + # INTERRUPT: talk over the assistant while its audio still streams. + # The serving overlap policy decides: enough overlapped speech barges + # in (the first response is cancelled) or a short remark is deferred + # until the model yields the turn. + print("[2] interrupting mid-answer ...", file=sys.stderr) + ids_before_interrupt = set(collector.response_ids) + await client.stream_pcm(interrupt, chunk_ms=chunk_ms) + await client.commit(create_response=False) + + # ANSWER AGAIN: wait for a response created *after* the interruption + # to finish. The first answer may still be streaming here; matching on + # any response.done would let its completion satisfy the wait and cut + # off the actual follow-up answer at close. + def follow_up_answer_done() -> bool: + return any( + event.get("type") == "response.done" + and collector.response_id(event) is not None + and collector.response_id(event) not in ids_before_interrupt + for event in collector.events + ) + + await wait_for_condition( + follow_up_answer_done, + timeout_s=args.timeout_s, + label="response.done for the response answering the interruption", + ) + + # HANG UP: report playback, close the session cleanly. + await acknowledge_collected_playback(client, collector) + await client.close(timeout_s=args.timeout_s) + try: + await asyncio.wait_for(consume_task, timeout=5.0) + except (asyncio.TimeoutError, asyncio.CancelledError): + consume_task.cancel() + + # Report: one WAV per response; statuses show what the overlap did. + responses = _fold_responses(collector) + summary: list[dict[str, object]] = [] + for index, (response_id, item) in enumerate(responses.items(), start=1): + audio = item["audio"] + assert isinstance(audio, bytes) + sample_rate_hz = int(item["sample_rate_hz"]) # type: ignore[arg-type] + wav_path = output_dir / f"response_{index}_{item['status']}.wav" + if audio: + write_pcm16_wav(wav_path, audio, sample_rate_hz=sample_rate_hz) + summary.append( + { + "response_id": response_id, + "status": item["status"], + "audio_s": round(len(audio) / (sample_rate_hz * 2), 2), + "text": item["text"], + "wav": wav_path.name if audio else None, + } + ) + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False)) + (output_dir / "events.jsonl").write_text( + "".join(json.dumps(event, ensure_ascii=False) + "\n" for event in collector.events), + encoding="utf-8", + ) + print(json.dumps(summary, indent=2, ensure_ascii=False)) + cancelled = sum(1 for item in summary if item["status"] == "cancelled") + print( + f"\n{len(summary)} responses ({cancelled} interrupted); outputs in {output_dir}", + file=sys.stderr, + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime") + parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") + parser.add_argument( + "--preset", + choices=["minicpmo_4_5", "personaplex", "none"], + default="minicpmo_4_5", + help="per-model session preset from vllm_omni.clients.", + ) + parser.add_argument("--ref-audio", help="reference voice WAV (the MiniCPM-o preset requires it)") + parser.add_argument("--question-wav", required=True, help="mono 16 kHz PCM16 question") + parser.add_argument("--interrupt-wav", required=True, help="mono 16 kHz PCM16 follow-up spoken mid-answer") + parser.add_argument("--output-dir", default="./duplex_out") + parser.add_argument( + "--chunk-ms", + type=int, + default=None, + help="append chunk size; defaults to the preset's model-unit-aligned size (minicpmo_4_5: 100, personaplex: 80)", + ) + parser.add_argument("--listen-s", type=float, default=2.0, help="how long to listen before interrupting") + parser.add_argument("--timeout-s", type=float, default=120.0) + return asyncio.run(run(parser.parse_args())) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/online_serving/barge_in_client_flow.md b/examples/online_serving/barge_in_client_flow.md new file mode 100644 index 000000000..d32a00e5a --- /dev/null +++ b/examples/online_serving/barge_in_client_flow.md @@ -0,0 +1,54 @@ +# `barge_in_client.py` — what happens, at a glance + +The demo plays one everyday conversation pattern against the full-duplex +server: ask something, get talked-over mid-answer, and see the assistant +cope. High-level flow below — for the exact wire events, read the +docstring in `barge_in_client.py`. + +```mermaid +sequenceDiagram + autonumber + participant U as user (client script) + participant A as assistant (server + model) + + Note over U,A: GREET + U->>A: start a voice session (with the reference voice) + A-->>U: ready + + Note over U,A: ASK + U->>A: ask a question (speak the question WAV) + Note right of A: decides on its own to answer
(no VAD — the model chooses) + A-->>U: starts answering aloud + + Note over U,A: INTERRUPT + Note over U: listens for ~2 seconds + par both talk at once + U->>A: interrupts with a follow-up + and + A-->>U: still speaking the first answer + end + + alt the interruption is substantial + Note right of A: stops mid-sentence (barge-in),
first answer is cut off + else it was just a short remark + Note right of A: finishes the first answer,
then takes the follow-up + end + + Note over U,A: ANSWER AGAIN + A-->>U: answers the follow-up aloud + + Note over U,A: HANG UP + U->>A: goodbye (close the session) + Note over U: saves one recording per answer
plus a summary of what happened +``` + +Outcome: the output directory shows which branch ran — a cut-off first +answer is saved as `response_1_cancelled.wav`, a completed one as +`response_1_completed.wav`, and `summary.json` records status, text, and +duration per answer. + +Companion: the runtime architecture lives in +[`docs/design/fullduplex.md`](../../docs/design/fullduplex.md); the client +API is `vllm_omni.clients.duplex.DuplexClient`, documented with the wire +protocol in +[`docs/serving/realtime_duplex_api.md`](../../docs/serving/realtime_duplex_api.md). diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 96c021a44..2836eeb92 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -4,9 +4,11 @@ This directory contains MiniCPM-o 4.5 online serving demos for vLLM-Omni. Inputs can include text, image, audio, or video; outputs are text and optional 24 kHz speech. -For the experimental native duplex runtime architecture, lifecycle invariants, +For the native duplex runtime architecture, lifecycle invariants, capability boundary, and validation scope, see -[`vllm_omni/experimental/fullduplex/DESIGN.md`](../../../vllm_omni/experimental/fullduplex/DESIGN.md). +[`docs/design/fullduplex.md`](../../../docs/design/fullduplex.md); for the +`DuplexClient` API and the `/v1/realtime?duplex=1` wire protocol, see +[`docs/serving/realtime_duplex_api.md`](../../../docs/serving/realtime_duplex_api.md). ## Installation @@ -231,7 +233,7 @@ Start the duplex generation server: ```bash vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code \ - --deploy-config vllm_omni/deploy/minicpmo_4_5_duplex.yaml \ + --deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \ --served-model-name openbmb/MiniCPM-o-4_5 \ --host 0.0.0.0 --port 8099 ``` diff --git a/examples/online_serving/minicpmo/barge_in_serve.sh b/examples/online_serving/minicpmo/barge_in_serve.sh new file mode 100644 index 000000000..6dade8c5c --- /dev/null +++ b/examples/online_serving/minicpmo/barge_in_serve.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +# Server side for barge_in_client.py: start MiniCPM-o 4.5 in native +# full-duplex mode. +# +# The duplex deploy profile registers the WS /v1/duplex endpoint (with +# /v1/realtime?duplex=1 projecting onto the same handler), enables the +# engine duplex control plane, and bounds the Thinker/Talker/Code2Wav +# stages to four live duplex sessions on one GPU. See +# vllm_omni/deploy/minicpmo_4_5.yaml (session_mode: duplex) for the session +# limits; every other duplex_session knob keeps its runtime default +# (idle TTL 300s, disconnect grace 30s, 16MiB pending input per session). +set -euo pipefail + +MODEL="${MODEL:-openbmb/MiniCPM-o-4_5}" +PORT="${PORT:-8099}" + +exec vllm-omni serve "$MODEL" \ + --omni \ + --deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \ + --trust-remote-code \ + --host 0.0.0.0 --port "$PORT" +# duplex endpoints: ws://:$PORT/v1/duplex and /v1/realtime?duplex=1 diff --git a/examples/online_serving/minicpmo/realtime_duplex_demo.py b/examples/online_serving/minicpmo/realtime_duplex_demo.py index b12c20a56..9cbc7f29b 100644 --- a/examples/online_serving/minicpmo/realtime_duplex_demo.py +++ b/examples/online_serving/minicpmo/realtime_duplex_demo.py @@ -23,23 +23,26 @@ REPO_ROOT = Path(__file__).resolve().parents[3] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from vllm_omni.experimental.fullduplex.client import ( # noqa: E402 +from vllm_omni.clients.duplex import ( # noqa: E402 PCM16_BYTES_PER_SAMPLE, PCM16_SAMPLE_RATE, - RealtimeDuplexClient, - RealtimeEventCollector, - build_realtime_url, + DuplexClient, + EventCollector, + acknowledge_collected_playback, read_pcm16_wav, - wait_for, + wait_for_condition, write_pcm16_wav, ) -from vllm_omni.experimental.fullduplex.client import chunk_period_ms as _chunk_period_ms # noqa: E402 -from vllm_omni.experimental.fullduplex.client import ( # noqa: E402 +from vllm_omni.clients.duplex import chunk_period_ms as _chunk_period_ms # noqa: E402 +from vllm_omni.clients.duplex import ( # noqa: E402 has_residual_model_unit as _has_residual_model_unit, ) -from vllm_omni.experimental.fullduplex.client import ( # noqa: E402 +from vllm_omni.clients.duplex import ( # noqa: E402 reference_audio_data_url as _ref_audio_data_url, ) +from vllm_omni.clients.minicpmo_4_5 import ( # noqa: E402 + create_duplex_session_config, +) from vllm_omni.experimental.fullduplex.video_stacking import ( # noqa: E402 concat_frames_b64, unit_subframe_offsets, @@ -96,7 +99,7 @@ class _StreamingOutputWriter: ) -class _StreamingEventCollector(RealtimeEventCollector): +class _StreamingEventCollector(EventCollector): def __init__(self, writer: _StreamingOutputWriter) -> None: super().__init__() self._writer = writer @@ -155,7 +158,7 @@ def _response_in_progress(events: list[dict[str, object]]) -> bool: ) -async def _commit_input_after_playback_checkpoint(client: RealtimeDuplexClient) -> float: +async def _commit_input_after_playback_checkpoint(client: DuplexClient, collector: EventCollector) -> float: """Commit input only after preserving already-played assistant history. Native duplex can finish several responses while the user is still @@ -166,7 +169,7 @@ async def _commit_input_after_playback_checkpoint(client: RealtimeDuplexClient) original history position. A second acknowledgement after the commit can then safely advance response-local playback that drained in the meantime. """ - await client.acknowledge_playback() + await acknowledge_collected_playback(client, collector) commit_sent_at_s = time.monotonic() await client.commit() return commit_sent_at_s @@ -434,87 +437,95 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]: if not input_pcm16: raise ValueError("input WAV has no audio") - url = build_realtime_url( + collector = _StreamingEventCollector(stream_writer) + client = DuplexClient( args.url, - args.model, - autostart=False if args.ref_audio else None, - session_id=args.session_id, - ) - client = RealtimeDuplexClient(url) - client.events = _StreamingEventCollector(stream_writer) - async with client: - await client.configure( - args.model, + model=args.model, + config=create_duplex_session_config( ref_audio=_ref_audio_data_url(args.ref_audio), - session_id=args.session_id, temperature=args.temperature, - timeout_s=args.timeout_s, - ) - stream_event_cursor = len(client.events.events) - frames_sent = await client.stream_pcm16( + ), + session_id=args.session_id, + reconnect=None, + heartbeat_interval_s=None, + handshake_timeout_s=args.timeout_s, + ) + async with client: + consume_task = asyncio.create_task(collector.consume(client)) + # The handshake consumed session.created before the collector + # subscribed; seed it so counts and capability lookups still work. + collector.add({"type": "session.created", "session": client.session_info}) + stream_event_cursor = len(collector.events) + frames_sent = await client.stream_pcm( input_pcm16, chunk_ms=args.chunk_ms, realtime=not args.no_realtime_pacing, video_frames=video_frames, stacked_video_frames=stacked_video_frames, ) - commit_event_cursor = len(client.events.events) - stream_decision = _latest_model_decision(client.events.events, stream_event_cursor) + commit_event_cursor = len(collector.events) + stream_decision = _latest_model_decision(collector.events, stream_event_cursor) input_has_residual_model_unit = _has_residual_model_unit( input_pcm16, - chunk_period_ms=_chunk_period_ms(client.events.events), + chunk_period_ms=_chunk_period_ms(collector.events), ) wait_for_post_commit_decision = False - commit_sent_at_s = await _commit_input_after_playback_checkpoint(client) + commit_sent_at_s = await _commit_input_after_playback_checkpoint(client, collector) wait_error: str | None = None committed_index: int | None = None post_commit_decision: str | None = None try: - await wait_for( - lambda: _input_committed_index(client.events.events, commit_event_cursor) is not None, + await wait_for_condition( + lambda: _input_committed_index(collector.events, commit_event_cursor) is not None, timeout_s=args.timeout_s, label="input_audio_buffer.committed", ) - committed_index = _input_committed_index(client.events.events, commit_event_cursor) - stream_decision = _latest_model_decision(client.events.events[: committed_index + 1], stream_event_cursor) + committed_index = _input_committed_index(collector.events, commit_event_cursor) + stream_decision = _latest_model_decision(collector.events[: committed_index + 1], stream_event_cursor) wait_for_post_commit_decision = input_has_residual_model_unit or _response_in_progress( - client.events.events[: committed_index + 1] + collector.events[: committed_index + 1] ) if wait_for_post_commit_decision: - await wait_for( - lambda: _post_commit_model_decision(client.events.events, committed_index) is not None, + await wait_for_condition( + lambda: _post_commit_model_decision(collector.events, committed_index) is not None, timeout_s=args.timeout_s, label="post-commit model decision or response drain", ) - post_commit_decision = _post_commit_model_decision(client.events.events, committed_index) + post_commit_decision = _post_commit_model_decision(collector.events, committed_index) except TimeoutError as exc: wait_error = str(exc) - await client.acknowledge_playback() + await acknowledge_collected_playback(client, collector) close_error: str | None = None + await client.close(timeout_s=args.timeout_s) + # close() returns when the client's reader observed session.closed; + # the collector is a separate subscriber task, so wait for it to + # drain before checking, or a normal close reads as a failure. try: - await client.close_session(timeout_s=args.timeout_s) - except TimeoutError as exc: - close_error = str(exc) + await asyncio.wait_for(consume_task, timeout=5.0) + except (asyncio.TimeoutError, asyncio.CancelledError): + consume_task.cancel() + if collector.count("session.closed") == 0: + close_error = "Timed out waiting for session.closed" - audio = client.events.audio_bytes() - first_text_at_s = client.events.first_received_at( + audio = collector.audio_bytes() + first_text_at_s = collector.first_received_at( "response.audio_transcript.delta", "response.output_text.delta", after_s=commit_sent_at_s, ) - first_audio_at_s = client.events.first_received_at( + first_audio_at_s = collector.first_received_at( "response.audio.delta", after_s=commit_sent_at_s, ) - response_created_at_s = client.events.first_received_at( + response_created_at_s = collector.first_received_at( "response.created", after_s=commit_sent_at_s, ) - response_done_at_s = client.events.first_received_at( + response_done_at_s = collector.first_received_at( "response.done", after_s=commit_sent_at_s, ) - audio_duration_s = len(audio) / (client.events.output_sample_rate_hz * 2) + audio_duration_s = len(audio) / (collector.output_sample_rate_hz * 2) response_generation_s = ( response_done_at_s - response_created_at_s if response_done_at_s is not None and response_created_at_s is not None @@ -522,27 +533,27 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]: ) transcript_deltas = [ str(event.get("delta", "")) - for event in client.events.events + for event in collector.events if event.get("type") in { "response.audio_transcript.delta", "response.output_text.delta", } ] - response_id = client.events.response_ids[0] if client.events.response_ids else None - timing = client.events.timing_summary( + response_id = collector.response_ids[0] if collector.response_ids else None + timing = collector.timing_summary( after_s=commit_sent_at_s, input_committed_at_s=commit_sent_at_s, response_id=response_id, ) - errors = client.events.errors() + errors = collector.errors() if wait_error: errors.append({"type": "client.timeout", "message": wait_error}) if close_error: errors.append({"type": "client.timeout", "message": close_error}) model_decision = post_commit_decision or stream_decision (output_dir / "events.jsonl").write_text( - "".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events), + "".join(json.dumps(event, ensure_ascii=False) + "\n" for event in collector.events), encoding="utf-8", ) (output_dir / "output.pcm").write_bytes(audio) @@ -550,13 +561,13 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]: write_pcm16_wav( output_dir / "output.wav", audio, - sample_rate_hz=client.events.output_sample_rate_hz, + sample_rate_hz=collector.output_sample_rate_hz, ) result = { "ok": ( - client.events.count("session.created") > 0 - and client.events.count("session.closed") > 0 + collector.count("session.created") > 0 + and collector.count("session.closed") > 0 and not errors and model_decision is not None and (bool(audio) or not args.require_audio) @@ -568,12 +579,12 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]: "decision_required": wait_for_post_commit_decision, "input_had_residual_model_unit": input_has_residual_model_unit, "response_listen_count": _event_count_after( - client.events.events, + collector.events, "response.listen", committed_index, ), "response_done_count": _event_count_after( - client.events.events, + collector.events, "response.done", committed_index, ), @@ -585,7 +596,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]: "audio_bytes": len(audio), "audio_chunk_count": len(stream_writer.audio_chunk_paths), "audio_chunk_files": [str(path) for path in stream_writer.audio_chunk_paths], - "output_sample_rate_hz": client.events.output_sample_rate_hz, + "output_sample_rate_hz": collector.output_sample_rate_hz, "latency": { "ttft_ms": ( round((first_text_at_s - commit_sent_at_s) * 1000, 2) if first_text_at_s is not None else None @@ -611,7 +622,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]: "measurement_origin": "input_audio_buffer.commit send", }, "timing": timing, - "response_ids": client.events.response_ids, + "response_ids": collector.response_ids, "transcript": "".join(transcript_deltas), "errors": errors, "output_dir": str(output_dir), diff --git a/examples/online_serving/minicpmo/realtime_web/app/static/app.js b/examples/online_serving/minicpmo/realtime_web/app/static/app.js index e8a3047c6..2943eec74 100644 --- a/examples/online_serving/minicpmo/realtime_web/app/static/app.js +++ b/examples/online_serving/minicpmo/realtime_web/app/static/app.js @@ -86,7 +86,7 @@ url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; url.searchParams.set('duplex', '1'); url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5'); - url.searchParams.set('minicpmo45_native_duplex', '1'); + url.searchParams.set('native_duplex', '1'); url.searchParams.set('autostart', '0'); return url.toString(); } @@ -458,7 +458,7 @@ settled = true; const extraBody = { auto_response: true, - minicpmo45_native_duplex: true, + native_duplex: true, }; const session = { modalities: ['audio', 'text'], diff --git a/examples/online_serving/personaplex/README.md b/examples/online_serving/personaplex/README.md index 3b737d275..f03b46908 100644 --- a/examples/online_serving/personaplex/README.md +++ b/examples/online_serving/personaplex/README.md @@ -1,47 +1,40 @@ # PersonaPlex full-duplex online serving Serve [`nvidia/personaplex-7b-v1`](https://huggingface.co/nvidia/personaplex-7b-v1) -(a Moshi-based full-duplex speech-to-speech model) with the native vLLM-Omni engine. +(a Moshi-based full-duplex speech-to-speech model) with the native vLLM-Omni engine +through the unified duplex serving stack (`/v1/duplex` and `/v1/realtime?duplex=1`). -The server hosts the **official PersonaPlex web client** at `/` (auto-downloaded from -the model repo) and implements its WebSocket protocol at `/api/chat`, so you talk to the -model live in the browser — mic in, agent speech + inner-monologue text out. - -> Experimental. Requires a GPU and Hugging Face access to the gated repo +> Requires a GPU and Hugging Face access to the gated repo > (`HF_TOKEN` with access to `nvidia/personaplex-7b-v1`). ## Start the server -```bash -HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 bash run_server.sh -# or directly: -HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m \ - vllm_omni.experimental.fullduplex.personaplex.serving.server --port 8124 -``` - -## Talk to it - -- **Browser (recommended):** open `http://localhost:8124/` and allow the microphone. - Use headphones so the agent does not hear itself. -- **Headless:** stream a 24 kHz mono WAV and save the reply: - - ```bash - python duplex_client.py --url ws://localhost:8124/api/chat --input user.wav --out reply.wav - ``` - -## Unified Realtime API - -The default `vllm_omni/deploy/personaplex.yaml` also enables main's engine-owned -full-duplex control plane: +The default `vllm_omni/deploy/personaplex.yaml` enables the engine-owned +full-duplex control plane (`session_mode: duplex`): ```bash -CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \ +HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \ /path/to/personaplex-7b-v1 \ --omni \ --deploy-config vllm_omni/deploy/personaplex.yaml ``` -Validate the actual `/v1/realtime?duplex=1` scheduler path with paced 24 kHz +This exposes: + +- `WS /v1/duplex` — the native duplex session dialect + (`session.create` / `input_audio_buffer.append` / `response.output_audio.delta` ...); +- `WS /v1/realtime?duplex=1` — the same sessions projected onto the + OpenAI Realtime event vocabulary (client API and wire protocol: + [`docs/serving/realtime_duplex_api.md`](../../../docs/serving/realtime_duplex_api.md)). + +PersonaPlex is a pure-lockstep model: every session on a PersonaPlex deployment is +native duplex (`is_enabled()` is unconditionally true), audio flows continuously in +both directions, and there are no client commits or external turn signals +(`supports_client_commit=false`, `supports_external_turn_signal=false`). + +## Validate the serving path + +Validate the `/v1/realtime?duplex=1` scheduler path with paced 24 kHz PCM, two concurrent sessions, overflow admission, per-session slot recycling, and non-silent output: @@ -52,33 +45,19 @@ python tests/e2e/online_serving/personaplex_realtime_duplex.py \ --output-dir /tmp/personaplex-realtime-duplex ``` -This is distinct from the standalone `/api/chat` compatibility server above. The unified endpoint advertises `supports_barge_in=false`: overlapping speech is native model behavior, but destructive output interruption and model-state rewind have not been validated for PersonaPlex. -## Protocol (`/api/chat`) - -Binary WebSocket messages, first byte is a tag (identical to `moshi.server`): - -| Direction | Tag | Payload | -| --- | --- | --- | -| server → client | `\x00` | ready handshake (after the system-prompt prefill) | -| client → server | `\x01` | Opus-encoded mic audio (24 kHz) | -| server → client | `\x01` | Opus-encoded agent audio | -| server → client | `\x02` | UTF-8 inner-monologue text | - -Connect with query params `text_prompt` (persona) and `voice_prompt` (voice file). -A raw-PCM endpoint (`/v1/audio/duplex`, JSON `open` + float32 frames) is also available -for clients that do not want an Opus dependency. - ## Notes -- **Run the client near the server.** Real-time 80 ms audio is sensitive to network - latency/jitter; over a high-latency remote link playback can stutter regardless of - engine speed. On localhost it is smooth. -- Greedy decoding by default. One conversation per server by default (a second - concurrent connection is rejected with WS close 1013 until the first ends); - pass `--batch-size N` to host N concurrent conversations on one engine - (elastic batching; connections beyond N are rejected until a slot frees). +- **Run the client near the server.** Real-time 80 ms frame audio is sensitive to + network latency/jitter; over a high-latency remote link playback can stutter + regardless of engine speed. On localhost it is smooth. +- The earlier standalone Moshi-web compatibility server (browser client at `/`, + binary WS protocol at `/api/chat`, raw-PCM `/v1/audio/duplex`) was demo-only + and has been removed; use the unified endpoints above. +- Session config (voice / persona / sampling) is passed per session via + `extra_body`; see + `vllm_omni/model_executor/models/personaplex/duplex/serving_adapter.py`. - Full runbook: [`recipes/NVIDIA/PersonaPlex.md`](../../../recipes/NVIDIA/PersonaPlex.md). diff --git a/examples/online_serving/personaplex/duplex_client.py b/examples/online_serving/personaplex/duplex_client.py deleted file mode 100644 index bb1eaba2b..000000000 --- a/examples/online_serving/personaplex/duplex_client.py +++ /dev/null @@ -1,114 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Headless client for the PersonaPlex duplex server (official Moshi protocol). - -Streams a 24 kHz mono WAV to ``/api/chat`` exactly as the browser client does -(Opus over WebSocket, tags ``\\x00`` ready / ``\\x01`` audio / ``\\x02`` text), -collects the agent's reply audio + inner-monologue text, and writes a WAV. - -The browser UI at ``http://:/`` is the primary demo; this script is -for headless testing / scripting. - - python duplex_client.py --url ws://localhost:8124/api/chat \ - --input user.wav --out reply.wav - -Deps: ``websockets``, ``sphn``, ``soundfile``, ``numpy``. -""" - -from __future__ import annotations - -import argparse -import asyncio - -import numpy as np -import soundfile as sf -import sphn - -SR = 24000 # PersonaPlex / Mimi codec rate - - -async def converse(url: str, pcm: np.ndarray) -> tuple[np.ndarray, str]: - import websockets - - reader = sphn.OpusStreamReader(SR) # decode agent audio - writer = sphn.OpusStreamWriter(SR) # encode our mic audio - text: list[str] = [] - ready = asyncio.Event() - last = {"t": 0.0} - - async with websockets.connect(url, max_size=None) as ws: - - async def receive() -> None: - loop = asyncio.get_event_loop() - while True: - try: - msg = await ws.recv() - except Exception: - return - if not isinstance(msg, (bytes, bytearray)) or not msg: - continue - tag, payload = msg[0], msg[1:] - if tag == 0: - ready.set() - elif tag == 1: - reader.append_bytes(bytes(payload)) - last["t"] = loop.time() - elif tag == 2: - text.append(payload.decode("utf8", errors="replace")) - - recv_task = asyncio.create_task(receive()) - await asyncio.wait_for(ready.wait(), timeout=120) # waits for system-prompt prefill - - frame = 1920 # 80 ms - for i in range(0, len(pcm), frame): - chunk = pcm[i : i + frame] - if len(chunk) < frame: - chunk = np.concatenate([chunk, np.zeros(frame - len(chunk), dtype=np.float32)]) - writer.append_pcm(chunk) - data = writer.read_bytes() - if data: - await ws.send(b"\x01" + data) - await asyncio.sleep(frame / SR) # pace at real time - - loop = asyncio.get_event_loop() - last["t"] = loop.time() - while loop.time() - last["t"] < 2.5: # drain until the agent stops talking - await asyncio.sleep(0.1) - recv_task.cancel() - - chunks: list[np.ndarray] = [] - while True: - out = reader.read_pcm() - if out is None or out.shape[-1] == 0: - break - chunks.append(np.asarray(out, dtype=np.float32)) - audio = np.concatenate(chunks) if chunks else np.zeros(0, dtype=np.float32) - return audio, "".join(text) - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--url", default="ws://localhost:8124/api/chat") - ap.add_argument("--input", required=True, help="24 kHz mono WAV of the user turn") - ap.add_argument("--out", default="personaplex_reply.wav") - ap.add_argument("--tail", type=float, default=10.0, help="trailing silence (s) for the agent reply") - ap.add_argument("--persona", default="") - ap.add_argument("--voice", default="NATF2.pt") - args = ap.parse_args() - - wav, sr = sf.read(args.input, dtype="float32") - if wav.ndim > 1: - wav = wav.mean(axis=1) - if sr != SR: - raise SystemExit(f"input must be {SR} Hz mono; got {sr} Hz. Resample it first.") - wav = np.concatenate([wav, np.zeros(int(args.tail * SR), dtype=np.float32)]) - - url = f"{args.url}?text_prompt={args.persona}&voice_prompt={args.voice}" - audio, text = asyncio.run(converse(url, wav)) - sf.write(args.out, audio if audio.size else np.zeros(SR, dtype=np.float32), SR) - print(f"agent reply: {audio.size / SR:.2f}s -> {args.out}") - print(f"inner-monologue: {text.strip()!r}") - - -if __name__ == "__main__": - main() diff --git a/examples/online_serving/personaplex/run_server.sh b/examples/online_serving/personaplex/run_server.sh deleted file mode 100755 index 53f6f0886..000000000 --- a/examples/online_serving/personaplex/run_server.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# PersonaPlex full-duplex online serving. -# -# Serves the OFFICIAL PersonaPlex web client at / (auto-downloaded from -# nvidia/personaplex-7b-v1) and the Moshi WebSocket protocol at /api/chat, -# backed by the native vLLM-Omni engine. Requires HF access to the gated repo. - -HOST="${HOST:-0.0.0.0}" -PORT="${PORT:-8124}" - -echo "Starting PersonaPlex duplex server on ${HOST}:${PORT}..." -echo "Open http://localhost:${PORT}/ for the official web client (run it near the server)." - -python -m vllm_omni.experimental.fullduplex.personaplex.serving.server \ - --host "$HOST" --port "$PORT" diff --git a/recipes/NVIDIA/PersonaPlex.md b/recipes/NVIDIA/PersonaPlex.md index 62d04a499..6f9e8544f 100644 --- a/recipes/NVIDIA/PersonaPlex.md +++ b/recipes/NVIDIA/PersonaPlex.md @@ -1,7 +1,7 @@ # PersonaPlex -> Live full-duplex speech-to-speech (talk to the model in your browser, it listens -> while it speaks) with persona and voice control, on the native vLLM-Omni engine. +> Live full-duplex speech-to-speech (the model listens while it speaks) with +> persona and voice control, on the native vLLM-Omni engine. ## Summary @@ -11,8 +11,8 @@ - Task: Full-duplex speech-to-speech. 24 kHz mic audio in, agent speech + inner-monologue text out, in 80 ms lockstep. Persona (role text) and voice (zero-shot clone) are set per session. -- Mode: Live duplex WebSocket server with the official browser client, single or - multi-session (`--batch-size`), plus an offline WAV-in / WAV-out example +- Mode: Live duplex WebSocket serving through the unified duplex stack + (`/v1/duplex` and `/v1/realtime?duplex=1`) - Maintainer: [`@linyueqian`](https://github.com/linyueqian) ## When to use this recipe @@ -24,23 +24,21 @@ streaming Mimi codec are all native vLLM-Omni modules (Mimi runs on matching the reference implementation frame for frame on the golden replays used as the acceptance gate. -PersonaPlex is the first Moshi-class (pure-lockstep) model in the experimental -full-duplex framework (`vllm_omni/experimental/fullduplex/`), and the first with -elastic multi-session batching: one engine hosts `--batch-size` concurrent -conversations, callers join or leave on any 80 ms tick without disturbing live -slots (greedy isolation is bit-exact). +PersonaPlex is the first Moshi-class (pure-lockstep) model on the vLLM-Omni +full-duplex serving stack: it plugs into the generic `/v1/duplex` handler +through the standard plugin seams (`duplex_serving_adapter` / +`duplex_runtime_extension` in its `pipeline.py`) with a model-specific +package at `vllm_omni/model_executor/models/personaplex/duplex/`. ## References -- Serving package: - [`vllm_omni/experimental/fullduplex/personaplex/`](../../vllm_omni/experimental/fullduplex/personaplex/) +- Duplex serving package: + [`vllm_omni/model_executor/models/personaplex/duplex/`](../../vllm_omni/model_executor/models/personaplex/duplex/) - Native model modules (temporal / depformer / streaming Mimi): [`vllm_omni/model_executor/models/personaplex/`](../../vllm_omni/model_executor/models/personaplex/) -- Online example (server + headless client): +- Online serving example: [`examples/online_serving/personaplex/`](../../examples/online_serving/personaplex/) -- Offline example: - [`examples/offline_inference/personaplex/personaplex_offline.py`](../../examples/offline_inference/personaplex/personaplex_offline.py) -- Staged offline pipeline (talker -> Mimi code2wav, async-chunk streaming): +- Staged pipeline (talker -> Mimi code2wav, async-chunk streaming): [`vllm_omni/deploy/personaplex.yaml`](../../vllm_omni/deploy/personaplex.yaml) - Integration PR: [vllm-project/vllm-omni#4771](https://github.com/vllm-project/vllm-omni/pull/4771) - Framework context: [RFC #3745](https://github.com/vllm-project/vllm-omni/issues/3745) @@ -53,18 +51,20 @@ slots (greedy isolation is bit-exact). ## Why it is different from MiniCPM-o / JoyVL duplex | | MiniCPM-o 4.5 (#3907) | JoyVL | **PersonaPlex (this)** | -|---|---|---|---| +| --- | --- | --- | --- | | Cadence | 1 s chunk groups | ~1 fps frames | **80 ms lockstep (12.5 Hz)** | | Turn control | learned `⟨listen⟩`/`⟨speak⟩` | ``/`` | **none, pure lockstep** | | Per step | variable-length token group | text decision | **1 user frame in -> 1 agent frame + 1 text token** | | Barge-in | at chunk boundary | n/a | **native (model always hears the user)** | | Session state | chunk-group KV | per-tick HTTP | **persistent ring KV + streaming Mimi state** | -This is the lockstep ("parallel-frame joint") shape of the duplex adapter -patterns: the adapter declares `DuplexCapability.continuous = True`, and -`core.DuplexRuntime` runs ONE eternal response that consumes input frames as they -arrive and drains on close. The flag is a small, default-off lifecycle mode, so -turn-based adapters (JoyVL, MiniCPM-o) are unaffected. +This is the lockstep ("parallel-frame joint") shape of full duplex: audio flows +continuously in both directions with no client commits or external turn signals. +On the unified stack this is expressed purely through the capability payload +(`supports_client_commit=false`, `supports_external_turn_signal=false`, +`supports_barge_in=false`); every session on a PersonaPlex deployment is native +duplex (`is_enabled()` is unconditionally true). Turn-based models +(MiniCPM-o 4.5) are unaffected. ## Architecture (Moshi RQ-Transformer, all native) @@ -95,9 +95,7 @@ native modules, so other CUDA GPUs with enough memory are expected to work. - OS: Linux - Python: 3.10+ (CI covers 3.11 / 3.12) -- vLLM-Omni: PR #4771 branch or current `main` once merged -- Extra dependency for the live server: `pip install sphn` (Opus framing; - `aiohttp` already ships with vLLM) +- vLLM-Omni: current `main` - `export HF_TOKEN=...` with access to the gated `nvidia/personaplex-7b-v1` repo (accept the license on the model page) @@ -107,67 +105,49 @@ First run auto-downloads from the model repo: `model.safetensors`, #### Command -Live duplex server (official browser client served at `/`): +Live duplex serving through the unified stack: ```bash -HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 \ -python -m vllm_omni.experimental.fullduplex.personaplex.serving.server \ - --port 8124 --voice NATF2.pt +HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \ + /path/to/personaplex-7b-v1 \ + --omni \ + --deploy-config vllm_omni/deploy/personaplex.yaml ``` -Multi-session on the same GPU (elastic batching, callers join/leave any tick): - -```bash -HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 \ -python -m vllm_omni.experimental.fullduplex.personaplex.serving.server \ - --port 8124 --batch-size 4 -``` - -Then open `http://localhost:8124/` and allow the microphone (use headphones so -the agent does not hear itself). Headless alternative: - -```bash -python examples/online_serving/personaplex/duplex_client.py \ - --url ws://localhost:8124/api/chat --input user.wav --out reply.wav -``` - -Offline WAV-in / WAV-out (no server): - -```bash -python examples/offline_inference/personaplex/personaplex_offline.py \ - --input-wav question.wav --output-wav reply.wav --output-text reply.json \ - --voice-prompt NATF2.pt --persona "You are a wise and friendly teacher." -``` +This exposes `WS /v1/duplex` (native duplex dialect) and +`WS /v1/realtime?duplex=1` (OpenAI Realtime projection; client API and wire +protocol in [`docs/serving/realtime_duplex_api.md`](../../docs/serving/realtime_duplex_api.md)). +Voice and persona are set per session via `extra_body`. #### Verification ```bash -# GPU-free contract tests (stubbed FrameStepper) -pytest tests/e2e/features/fullduplex/test_personaplex_adapter.py \ - tests/e2e/features/fullduplex/test_personaplex_session.py \ - tests/e2e/features/fullduplex/test_personaplex_batched.py -q +# GPU-free contract tests (stage0 runtime + unified serving adapter) +pytest tests/model_executor/models/personaplex/duplex/ -q -# GPU e2e: batched isolation + slot recycle (coherence check needs a real -# question WAV via PPLEX_QUESTION_WAV) -pytest tests/e2e/features/fullduplex/test_personaplex_elastic.py -q +# GPU e2e: paced 24 kHz PCM over /v1/realtime?duplex=1, two concurrent +# sessions, overflow admission, slot recycling, non-silent output +python tests/e2e/online_serving/personaplex_realtime_duplex.py \ + --model /path/to/personaplex-7b-v1 \ + --input-wav /path/to/speech.wav \ + --output-dir /tmp/personaplex-realtime-duplex ``` -A green offline run producing a coherent spoken reply (and inner-monologue text -in `reply.json`) is the quickest end-to-end check. - #### Notes -- Realtime budget is 80 ms/frame. Verified eager per-tick latency at - `--batch-size 4` is ~70-74 ms on this hardware class, i.e. all four +- Realtime budget is 80 ms/frame. Verified eager per-tick latency at four + concurrent sessions is ~70-74 ms on this hardware class, i.e. all four conversations stay realtime; single-session has comfortable headroom. - Decoding is greedy only (temperature/top-k knobs are intentionally not exposed): greedy is what the parity gates pin against the reference implementation, and sampling amplifies sub-bit numeric drift into divergence. -- With `--batch-size N`, connections beyond N are rejected until a slot frees; - a recycled slot is bit-exact with a fresh engine for the same inputs. +- Session concurrency is governed by the deploy config + (`vllm_omni/deploy/personaplex.yaml`); connections beyond the limit are + rejected until a slot frees. A recycled slot is bit-exact with a fresh + engine for the same inputs. - Voice prompts: bundled `voices.tgz` names (`NATF*`/`NATM*` natural, `VARF*`/`VARM*` varied) or a path to your own `.pt`/`.wav`. -- Run the browser client near the server. The 80 ms cadence is sensitive to +- Run the client near the server. The 80 ms cadence is sensitive to network jitter; over a high-latency link playback can stutter regardless of engine speed. On localhost it is smooth. - Sliding window is 3000 frames (~4 min). The ring KV recycles beyond that; diff --git a/tests/benchmarks/patch/test_patch.py b/tests/benchmarks/patch/test_patch.py index d5c714be8..e72d5a263 100644 --- a/tests/benchmarks/patch/test_patch.py +++ b/tests/benchmarks/patch/test_patch.py @@ -28,7 +28,7 @@ from vllm_omni.benchmarks.patch.patch import ( async_request_openai_realtime_duplex, should_request_stage_metrics, ) -from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector +from vllm_omni.clients.duplex import EventCollector pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu] @@ -131,7 +131,7 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch) def __init__(self, url): assert url == "ws://localhost:8000/v1/realtime?duplex=1" - self.events = RealtimeEventCollector() + self.events = EventCollector() self.configure_kwargs = None self.sent = [] self.response_count = 0 @@ -206,7 +206,7 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch) return None monkeypatch.setattr( - "vllm_omni.benchmarks.patch.patch.RealtimeDuplexClient", + "vllm_omni.benchmarks.patch.patch._RealtimeTTSProbe", FakeRealtimeClient, ) request_input = RequestFuncInput( diff --git a/tests/benchmarks/test_omniinteract.py b/tests/benchmarks/test_omniinteract.py index c51724e0a..f9488b11c 100644 --- a/tests/benchmarks/test_omniinteract.py +++ b/tests/benchmarks/test_omniinteract.py @@ -25,9 +25,9 @@ from vllm_omni.benchmarks import serve as benchmark_serve from vllm_omni.benchmarks.data_modules import omniinteract_dataset as data from vllm_omni.benchmarks.metrics.metrics import calculate_metrics from vllm_omni.benchmarks.patch import patch as benchmark_patch +from vllm_omni.clients.duplex import EventCollector from vllm_omni.entrypoints.cli.benchmark.cli_args import preprocess_serve_args from vllm_omni.entrypoints.cli.benchmark.serve import OmniBenchmarkServingSubcommand -from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector from vllm_omni.utils.tracking_parser import TrackingArgumentParser pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark] @@ -42,8 +42,8 @@ def _case(tmp_path: Path, *, subset: str = "1q1a", name: str = "video.mp4") -> d return data.OmniInteractCase(subset, name, video, annotation, "multi_turn") -def _collector(*events: tuple[dict[str, object], float]) -> RealtimeEventCollector: - collector = RealtimeEventCollector() +def _collector(*events: tuple[dict[str, object], float]) -> EventCollector: + collector = EventCollector() for event, received_at in events: collector.add(event, received_at_s=received_at) return collector @@ -136,7 +136,7 @@ def _successful_output( *, audio_time: float = 10.0, status: str = "completed", -) -> tuple[data.OmniInteractCase, RealtimeEventCollector, oi.OmniInteractCaseResult]: +) -> tuple[data.OmniInteractCase, EventCollector, oi.OmniInteractCaseResult]: case = _case(tmp_path) collector = _collector( (_created(), 9.9), @@ -151,7 +151,7 @@ def _successful_output( def _write_success( root: Path, case: data.OmniInteractCase, - collector: RealtimeEventCollector, + collector: EventCollector, result: oi.OmniInteractCaseResult, **kwargs, ): @@ -285,7 +285,7 @@ def test_response_ledger_rejects_identity_errors(events, match: str): class _CompletionClient: - def __init__(self, collector: RealtimeEventCollector): + def __init__(self, collector: EventCollector): self.events, self.acks = collector, [] def raise_if_reader_stopped(self) -> None: @@ -334,13 +334,71 @@ async def test_deferred_commit_ignores_the_prior_response_terminal(): @pytest.mark.asyncio -async def test_playback_acks_only_after_serial_audio_drain(): - collector = _collector((_audio(samples=2400), 10.0), (_done(), 10.01)) +async def test_playback_acks_incrementally_along_the_serial_clock(): + # 1 s of audio playing from t=10.0: the first ack checkpoints the + # response as soon as audio arrives, later acks report cumulative + # progress at least every _ACK_STEP_MS of new audio, and the completion + # ack lands exactly at the total once the serial clock drains it. + collector = _collector((_audio(samples=24_000), 10.0), (_done(), 10.01)) client, playback = _CompletionClient(collector), oi._Playback() await playback.acknowledge(client, now=10.05) - assert client.acks == [] + assert client.acks == [("r1", 50)] # checkpoint ack, reserves the slot await playback.acknowledge(client, now=10.1) - assert client.acks == [("r1", 100)] + assert client.acks == [("r1", 50)] # step not reached, no spam + await playback.acknowledge(client, now=10.3) + assert client.acks == [("r1", 50), ("r1", 300)] + await playback.acknowledge(client, now=11.0) + assert client.acks == [("r1", 50), ("r1", 300), ("r1", 1000)] + assert playback.completion_acked == {"r1"} + await playback.acknowledge(client, now=11.5) + assert client.acks[-1] == ("r1", 1000) # completion acked once + + +@pytest.mark.asyncio +async def test_playback_checkpoint_ack_fires_before_audio_plays(): + # Audio queued but none played yet: a 0 ms checkpoint ack still goes out + # so a later input commit cannot displace the response's history slot. + collector = _collector((_audio(samples=24_000), 10.0), (_done(), 10.01)) + client, playback = _CompletionClient(collector), oi._Playback() + await playback.acknowledge(client, now=10.0) + assert client.acks == [("r1", 0)] + + +@pytest.mark.asyncio +async def test_acks_continue_after_committed_input(): + # A later committed user input must not stop the acks: the checkpointed + # response's history slot is updated in place, so the drain ack for the + # tail is still delivered afterwards. + collector = _collector( + (_audio(samples=24_000), 10.0), + (_done(), 10.01), + ({"type": "input_audio_buffer.committed"}, 10.4), + ) + client, playback = _CompletionClient(collector), oi._Playback() + await playback.acknowledge(client, now=10.05) + assert client.acks == [("r1", 50)] + await playback.acknowledge(client, now=11.5) + assert client.acks == [("r1", 50), ("r1", 1000)] + assert playback.completion_acked == {"r1"} + + +def test_tolerated_playback_ack_rejection_is_a_warning_not_a_failure(): + collector = _collector( + ( + { + "type": "error", + "error": {"type": "invalid_request_error", "code": "playback_ack_too_late"}, + }, + 1.0, + ), + ) + warnings: list[str] = [] + oi._raise_if_session_terminated(collector, 0, warnings=warnings) + assert warnings == ["tolerated in-flight server rejection: playback_ack_too_late"] + + collector.add({"type": "error", "error": {"code": "bad_event", "message": "boom"}}, received_at_s=1.1) + with pytest.raises(RuntimeError, match="bad_event"): + oi._raise_if_session_terminated(collector, 0, warnings=warnings) @pytest.mark.parametrize( @@ -421,30 +479,34 @@ def test_publication_failure_removes_partial_success_bundle(tmp_path: Path, monk def test_atomic_write_preserves_destination_on_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): destination = tmp_path / "output.wav" destination.write_bytes(b"old") - monkeypatch.setattr(oi, "write_pcm16_wav", lambda *a, **k: (_ for _ in ()).throw(OSError("write"))) + monkeypatch.setattr( + "vllm_omni.clients.duplex.write_pcm16_wav", + lambda *a, **k: (_ for _ in ()).throw(OSError("write")), + ) with pytest.raises(OSError): oi._atomic_write_wav(destination, b"\0\0", 24_000) assert destination.read_bytes() == b"old" assert not list(tmp_path.glob(".*.tmp")) -class _RealtimeClient: +class _RealtimeClient(oi._RealtimeSession): + """Fake transport over the real session wrapper: URL and session-config + construction stay real, only the network-facing calls are stubbed.""" + instances: list[_RealtimeClient] = [] - def __init__(self, url: str, **kwargs): - self.url, self.events, self.acks, self.configure_kwargs = url, RealtimeEventCollector(), [], {} + def __init__(self, config: oi.OmniInteractBenchmarkConfig, session_id: str, reference_audio: str): + super().__init__(config, session_id, reference_audio) + self.acks: list[tuple[str, int]] = [] self.instances.append(self) async def __aenter__(self): + self.events.add({"type": "session.created", "session": {"capabilities": {"chunk_period_ms": 1000}}}) return self async def __aexit__(self, *args): return None - async def configure(self, model: str, **kwargs) -> None: - self.configure_kwargs = kwargs - self.events.add({"type": "session.created", "session": {"capabilities": {"chunk_period_ms": 1000}}}) - async def send(self, event: dict[str, object]) -> None: return None @@ -480,7 +542,7 @@ async def test_public_runner_executes_one_prepared_session(tmp_path: Path, monke extra_body={"custom": "value"}, ) prepared = data.OmniInteractPreparedInput(1.0, b"\0\0\0\0", ("frame",), "data:audio/wav;base64,ref") - monkeypatch.setattr(oi, "RealtimeDuplexClient", _RealtimeClient) + monkeypatch.setattr(oi, "_RealtimeSession", _RealtimeClient) monkeypatch.setattr(oi, "_COMPLETION_SETTLE_S", 0.0) result = await oi.run_omniinteract_case( case, @@ -491,8 +553,15 @@ async def test_public_runner_executes_one_prepared_session(tmp_path: Path, monke assert result.success and result.transcript == "answer" and result._artifact_context is not None assert not config.output_root.exists() assert "autostart=0" in _RealtimeClient.instances[-1].url - assert _RealtimeClient.instances[-1].configure_kwargs["extra_body"] == {"custom": "value"} - assert _RealtimeClient.instances[-1].acks == [("r1", 1)] + session_config = _RealtimeClient.instances[-1].session_config + assert session_config.extra_body["custom"] == "value" + assert session_config.extra_body["native_duplex"] is True + assert session_config.ref_audio == "data:audio/wav;base64,ref" + acks = _RealtimeClient.instances[-1].acks + # Cumulative acks for the one response: an optional 0 ms checkpoint the + # moment its audio arrived, then the exact total once drained. + assert acks[-1] == ("r1", 1) + assert acks in ([("r1", 1)], [("r1", 0), ("r1", 1)]) def test_standard_sample_loading_prepares_media_before_timing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): @@ -521,7 +590,12 @@ def test_standard_sample_loading_prepares_media_before_timing(tmp_path: Path, mo ) preprocess_serve_args(args) assert args.endpoint == "/v1/realtime" - monkeypatch.setattr(benchmark_patch, "reference_audio_data_url", lambda _: "data:audio/wav;base64,ref") + # The benchmark patch module imports the duplex client library lazily + # (it must stay out of the CLI import graph), so patch the provider. + monkeypatch.setattr( + "vllm_omni.clients.duplex.reference_audio_data_url", + lambda _: "data:audio/wav;base64,ref", + ) monkeypatch.setattr(benchmark_patch, "prepare_media", lambda *a, **k: (1.0, b"pcm", ["frame"])) samples = benchmark_patch.get_samples(args, None) assert len(samples) == args.num_prompts == 3 diff --git a/tests/clients/__init__.py b/tests/clients/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/tests/clients/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/tests/clients/test_duplex_client.py b/tests/clients/test_duplex_client.py new file mode 100644 index 000000000..1be3eb943 --- /dev/null +++ b/tests/clients/test_duplex_client.py @@ -0,0 +1,895 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Unit tests for the public duplex client (fake transport, no server).""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import wave +from urllib.parse import parse_qs, urlsplit + +import pytest + +from vllm_omni.clients.duplex import ( + AudioFormat, + ConnectionResumed, + DuplexClient, + DuplexProtocolError, + DuplexSessionClosedError, + EventCollector, + ReconnectPolicy, + SessionConfig, + SessionResumed, + build_realtime_url, + chunk_period_ms, + duplex_unit_boundary_ms, + has_residual_model_unit, + read_pcm16_wav, + reference_audio_data_url, + summarize_session_request_metrics, + write_pcm16_wav, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +SESSION_CREATED = { + "type": "session.created", + "session": {"session_id": "sess-1"}, + "incarnation": 0, + "resume_token": "tok-1", + "server_event_seq": 1, +} +SESSION_CLOSED = {"type": "session.closed", "session_id": "sess-1", "server_event_seq": 99} + + +class FakeSocket: + def __init__(self) -> None: + self.sent: list[dict[str, object]] = [] + self.incoming: asyncio.Queue = asyncio.Queue() + self.closed = False + + async def send(self, raw: str) -> None: + if self.closed: + raise RuntimeError("socket closed") + self.sent.append(json.loads(raw)) + + async def recv(self) -> str: + item = await self.incoming.get() + if isinstance(item, BaseException): + raise item + return json.dumps(item) + + async def close(self) -> None: + self.closed = True + + def feed(self, event: dict[str, object] | BaseException) -> None: + self.incoming.put_nowait(event) + + def sent_types(self) -> list[object]: + return [event.get("type") for event in self.sent] + + +def make_client(*sockets: FakeSocket, **kwargs) -> tuple[DuplexClient, list[str]]: + remaining = list(sockets) + calls: list[str] = [] + + async def connect(url: str): + calls.append(url) + if not remaining: + raise ConnectionError("no more sockets") + return remaining.pop(0) + + kwargs.setdefault("heartbeat_interval_s", None) + kwargs.setdefault("reconnect", None) + kwargs.setdefault("handshake_timeout_s", 5.0) + client = DuplexClient("ws://test-host:8099", model="test-model", connect=connect, **kwargs) + return client, calls + + +# --------------------------------------------------------------------------- +# SessionConfig + + +def test_session_config_payload_defaults(): + payload = SessionConfig().to_session_payload(model="m", session_id="s") + assert payload["model"] == "m" + assert payload["session_id"] == "s" + assert payload["modalities"] == ["audio", "text"] + assert payload["input_audio_format"] == "pcm16" + assert payload["output_audio_format"] == "pcm16" + # The encodings do not pin the rates; the payload must carry both (the + # server reads the input rate top-level and the output rate from audio). + assert payload["sample_rate_hz"] == 16_000 + assert payload["audio"] == { + "input": {"sample_rate_hz": 16_000}, + "output": {"sample_rate_hz": 24_000}, + } + assert payload["turn_detection"] is None + assert payload["extra_body"] == {"auto_response": True} + assert "voice" not in payload + assert "ref_audio" not in payload + + +def test_session_config_payload_carries_preset_rates(): + config = SessionConfig( + input_audio=AudioFormat("pcm_f32le", 24_000), + output_audio=AudioFormat("pcm16", 24_000), + ) + payload = config.to_session_payload(model="m") + assert payload["input_audio_format"] == "pcm_f32le" + assert payload["sample_rate_hz"] == 24_000 + assert payload["audio"] == { + "input": {"sample_rate_hz": 24_000}, + "output": {"sample_rate_hz": 24_000}, + } + + +def test_audio_format_math(): + fmt = AudioFormat("pcm16", 16_000) + assert fmt.byte_count(100) == 3200 + assert fmt.duration_ms(3200) == 100.0 + f32 = AudioFormat("pcm_f32le", 24_000) + assert f32.byte_count(80) == 1920 * 4 + with pytest.raises(ValueError): + _ = AudioFormat("mp3", 16_000).bytes_per_sample + + +# --------------------------------------------------------------------------- +# Handshake and lifecycle + + +async def test_handshake_adopts_session_and_acks(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, calls = make_client(sock) + async with client: + assert client.session_id == "sess-1" + assert client.resume_token == "tok-1" + assert calls == ["ws://test-host:8099/v1/realtime?duplex=1&model=test-model&autostart=0"] + assert sock.sent[0]["type"] == "session.update" + assert sock.sent[0]["session"]["model"] == "test-model" + await _drain(lambda: {"type": "session.event_ack", "server_event_seq": 1} in _acks(sock)) + close_task = asyncio.create_task(client.close()) + await _drain(lambda: "session.close" in sock.sent_types()) + sock.feed(SESSION_CLOSED) + await close_task + assert "session.close" in sock.sent_types() + + +async def test_handshake_error_raises_protocol_error(): + sock = FakeSocket() + sock.feed({"type": "error", "error": {"code": "unsupported_audio_format", "message": "bad"}}) + client, _ = make_client(sock) + with pytest.raises(DuplexProtocolError) as excinfo: + async with client: + pass + assert excinfo.value.code == "unsupported_audio_format" + + +async def test_session_expired_raises_from_event_stream(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + sock.feed({"type": "session.expired", "reason": "lease_expired", "server_event_seq": 2}) + with pytest.raises(DuplexSessionClosedError) as excinfo: + async for _ in client.events(): + pass + assert "lease_expired" in excinfo.value.reason + + +async def test_resync_required_drops_resume_credential(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, calls = make_client(sock, reconnect=ReconnectPolicy(max_attempts=2, backoff_s=(0.0, 0.0))) + async with client: + assert client.resume_token == "tok-1" + stream = client.events() + sock.feed({"type": "session.resync_required", "session_id": "sess-1", "server_event_seq": 2}) + async for event in stream: + if event.type == "session.resync_required": + break + assert client.resume_token is None + # With the credential gone, a transport drop must finalize instead of + # attempting session.resume against a server that stopped journaling. + sock.feed(RuntimeError("transport dropped")) + with pytest.raises(DuplexSessionClosedError): + async for _ in stream: + pass + assert len(calls) == 1 + + +# --------------------------------------------------------------------------- +# Input events + + +async def test_append_audio_tracks_cumulative_end_ms(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + pcm = b"\x01\x02" * 1600 # 100 ms of pcm16 @ 16 kHz + await client.append_audio(pcm) + await client.append_audio(pcm, is_speech=True) + appends = [event for event in sock.sent if event.get("type") == "input_audio_buffer.append"] + assert [a["audio_end_ms"] for a in appends] == [100, 200] + assert appends[0]["format"] == "pcm16" + assert appends[0]["sample_rate_hz"] == 16_000 + assert appends[0]["duration_ms"] == 100 + assert "is_speech" not in appends[0] + assert appends[1]["is_speech"] is True + assert base64.b64decode(appends[0]["audio"]) == pcm + sock.feed(SESSION_CLOSED) + + +async def test_append_audio_strips_video_frame_data_url_prefix(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + jpeg_b64 = base64.b64encode(b"\xff\xd8fake").decode("ascii") + # The wire contract carries bare base64; image_data_url output must + # still be accepted (the server validator rejects data-URL prefixes). + await client.append_audio(b"\x00\x00", video_frames=[f"data:image/jpeg;base64,{jpeg_b64}", jpeg_b64]) + append = next(event for event in sock.sent if event.get("type") == "input_audio_buffer.append") + assert append["video_frames"] == [jpeg_b64, jpeg_b64] + sock.feed(SESSION_CLOSED) + + +async def test_stream_pcm_chunking(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + await client.stream_pcm(b"\x00" * 8000, chunk_ms=100, realtime=False) + appends = [event for event in sock.sent if event.get("type") == "input_audio_buffer.append"] + assert [a["duration_ms"] for a in appends] == [100, 100, 50] + sock.feed(SESSION_CLOSED) + + +async def test_interruption_primitives_send_documented_events(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + await client.cancel_response() + await client.clear_input() + types = sock.sent_types() + assert types.index("response.cancel") < types.index("input_audio_buffer.clear") + sock.feed(SESSION_CLOSED) + + +# --------------------------------------------------------------------------- +# Response demultiplexing + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +async def test_response_handle_flow(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + chunk = b"\x00\x01" * 2400 # 100 ms of pcm16 @ 24 kHz + client, _ = make_client(sock) + async with client: + sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2}) + sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3}) + sock.feed( + { + "type": "response.audio.delta", + "response_id": "resp-1", + "delta": _b64(chunk), + "sample_rate_hz": 24_000, + "server_event_seq": 4, + } + ) + sock.feed( + { + "type": "response.audio_transcript.delta", + "response_id": "resp-1", + "delta": "hi there", + "server_event_seq": 5, + } + ) + sock.feed({"type": "response.done", "response_id": "resp-1", "server_event_seq": 6}) + + async for response in client.responses(): + chunks = [piece async for piece in response.audio()] + assert chunks == [chunk] + assert response.decision == "speak" + assert response.transcript == "hi there" + assert response.played_ms == pytest.approx(100.0) + assert response.finished + break + sock.feed(SESSION_CLOSED) + + +async def test_listen_decision_yields_finished_silent_handle(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + sock.feed({"type": "response.listen", "server_event_seq": 2}) + async for response in client.responses(): + assert response.decision == "listen" + assert response.finished + assert [piece async for piece in response.audio()] == [] + break + sock.feed(SESSION_CLOSED) + + +async def test_listen_terminates_active_response(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2}) + sock.feed({"type": "response.listen", "response_id": "resp-1", "server_event_seq": 3}) + async for response in client.responses(): + await response.wait(timeout_s=5.0) + assert response.decision == "listen" + break + sock.feed(SESSION_CLOSED) + + +async def test_id_less_listen_never_closes_an_in_flight_response(): + # An id-less listen is a standalone decision beat (e.g. a silence-skip + # while a response streams); it must surface as its own finished handle + # and leave the in-flight response to its real terminal. + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2}) + sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3}) + sock.feed({"type": "response.listen", "server_event_seq": 4}) + sock.feed({"type": "response.done", "response_id": "resp-1", "server_event_seq": 5}) + seen: list[tuple[str | None, str | None, bool]] = [] + async for response in client.responses(): + await response.wait(timeout_s=5.0) + seen.append((response.response_id, response.decision, response.finished)) + if len(seen) == 2: + break + assert ("resp-1", "speak", True) in seen + assert (None, "listen", True) in seen + sock.feed(SESSION_CLOSED) + + +async def test_terminal_listen_keeps_spoken_decision(): + # A terminal listen after the response spoke is the model yielding the + # turn; the handle must stay decision="speak" with its audio intact. + sock = FakeSocket() + sock.feed(SESSION_CREATED) + chunk = b"\x00\x01" * 2400 + client, _ = make_client(sock) + async with client: + sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2}) + sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3}) + sock.feed( + { + "type": "response.audio.delta", + "response_id": "resp-1", + "delta": _b64(chunk), + "sample_rate_hz": 24_000, + "server_event_seq": 4, + } + ) + sock.feed({"type": "response.listen", "response_id": "resp-1", "server_event_seq": 5}) + async for response in client.responses(): + chunks = [piece async for piece in response.audio()] + assert chunks == [chunk] + assert response.decision == "speak" + assert response.finished + break + sock.feed(SESSION_CLOSED) + + +async def test_error_event_surfaces_on_response_path(): + # A rejected send produces no response; a consumer waiting in responses() + # must see the rejection instead of waiting forever. + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + sock.feed( + { + "type": "error", + "error": {"code": "input_audio_buffer_empty", "message": "empty commit"}, + "server_event_seq": 2, + } + ) + with pytest.raises(DuplexProtocolError) as excinfo: + async for _ in client.responses(): + pass + assert excinfo.value.code == "input_audio_buffer_empty" + sock.feed(SESSION_CLOSED) + + +async def test_slow_audio_consumer_drops_oldest_instead_of_stalling(): + from vllm_omni.clients.duplex import AudioDelta, ResponseHandle + + handle = ResponseHandle( + response_id="r1", + output_format=AudioFormat("pcm16", 24_000), + max_buffered_events=2, + ) + for payload in (b"c1", b"c2", b"c3"): + # _feed must never block the reader, even against a full queue. + handle._feed(AudioDelta({"type": "response.audio.delta", "delta": _b64(payload)})) + handle._finish(None) + chunks = [chunk async for chunk in handle.audio()] + assert chunks == [b"c3"] # oldest chunks were dropped, the sentinel landed + + +# --------------------------------------------------------------------------- +# Resume + + +async def test_resume_after_transport_drop(): + first = FakeSocket() + first.feed(SESSION_CREATED) + second = FakeSocket() + # Mirror the real resume activation payload + # (entrypoints/duplex/serving.py, activation_payload_factory): it carries + # no nested "session" object, only the identity fields. + second.feed( + { + "type": "session.resumed", + "session_id": "sess-1", + "incarnation": 0, + "attachment_generation": 1, + "resume_token": "tok-2", + "server_event_seq": 5, + } + ) + client, calls = make_client( + first, + second, + reconnect=ReconnectPolicy(max_attempts=2, backoff_s=(0.0, 0.0)), + ) + async with client: + stream = client.events() + + async def take_two(): + seen = [] + async for event in stream: + seen.append(event) + if len(seen) == 2: + return seen + + consumer = asyncio.create_task(take_two()) + await asyncio.sleep(0) # let the consumer subscribe before the drop + await asyncio.sleep(0) + first.feed(RuntimeError("transport dropped")) + seen = await asyncio.wait_for(consumer, timeout=5.0) + assert isinstance(seen[0], ConnectionResumed) + assert isinstance(seen[1], SessionResumed) + assert client.resume_token == "tok-2" + # The activation payload has no session object; the info captured at + # the original handshake must survive the resume. + assert client.session_info == {"session_id": "sess-1"} + assert len(calls) == 2 + resume = second.sent[0] + assert resume["type"] == "session.resume" + assert resume["session_id"] == "sess-1" + assert resume["incarnation"] == 0 + assert resume["resume_token"] == "tok-1" + assert resume["last_received_server_event_seq"] == 1 + second.feed(SESSION_CLOSED) + + +async def test_no_reconnect_policy_surfaces_closed(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) # reconnect=None + async with client: + stream = client.events() + sock.feed(RuntimeError("transport dropped")) + with pytest.raises(DuplexSessionClosedError): + async for _ in stream: + pass + + +async def test_heartbeat_survives_transient_send_failure(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + armed = {"fail": True} + original_send = sock.send + + async def flaky_send(raw: str) -> None: + if json.loads(raw).get("type") == "session.heartbeat" and armed["fail"]: + armed["fail"] = False + raise RuntimeError("transport hiccup") + await original_send(raw) + + sock.send = flaky_send # type: ignore[method-assign] + client, _ = make_client(sock, heartbeat_interval_s=0.01) + async with client: + # The first tick fails; the loop must keep ticking (a resumed but + # quiet session relies on heartbeats to reset the server timeout). + await _drain(lambda: "session.heartbeat" in sock.sent_types()) + sock.feed(SESSION_CLOSED) + + +async def test_aexit_sends_session_close_on_error_path(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + with pytest.raises(RuntimeError): + async with client: + raise RuntimeError("application failure") + # The server must not be left holding the session for the disconnect + # grace period; the error path still announces the close. + assert "session.close" in sock.sent_types() + + +def test_target_url_forces_autostart_off(): + client = DuplexClient("ws://test-host:8099/v1/realtime?autostart=1", model="m") + query = parse_qs(urlsplit(client._target_url()).query) + # autostart would race the session.update handshake and silently drop + # ref_audio/extra_body; the client overrides it even when the URL asks. + assert query["autostart"] == ["0"] + + +async def test_fatal_resume_error_gives_up(): + first = FakeSocket() + first.feed(SESSION_CREATED) + second = FakeSocket() + second.feed({"type": "error", "error": {"code": "invalid_resume_token", "message": "nope"}}) + client, calls = make_client( + first, + second, + reconnect=ReconnectPolicy(max_attempts=3, backoff_s=(0.0, 0.0)), + ) + async with client: + stream = client.events() + first.feed(RuntimeError("transport dropped")) + with pytest.raises(DuplexSessionClosedError): + async for _ in stream: + pass + assert len(calls) == 2 # no retry after a fatal resume error + + +# --------------------------------------------------------------------------- +# Collector + + +def test_event_collector_accumulates_audio(): + collector = EventCollector() + collector.add({"type": "response.created", "response": {"id": "r1"}}, received_at_s=1.0) + collector.add( + {"type": "response.audio.delta", "response_id": "r1", "delta": _b64(b"ab"), "sample_rate_hz": 16_000}, + received_at_s=1.1, + ) + collector.add( + {"type": "response.output_audio.delta", "response_id": "r1", "audio": _b64(b"cd")}, + received_at_s=1.2, + ) + assert collector.count("response.created") == 1 + assert collector.audio_bytes() == b"abcd" + assert collector.output_sample_rate_hz == 16_000 + summary = collector.timing_summary(after_s=0.0) + assert summary["audio_output"]["chunk_count"] == 2 + assert summary["audio_output"]["response_created_to_first_audio_ms"] == pytest.approx(100.0) + + +# --------------------------------------------------------------------------- +# helpers + + +def _acks(sock: FakeSocket) -> list[dict[str, object]]: + return [event for event in sock.sent if event.get("type") == "session.event_ack"] + + +async def _drain(predicate, *, timeout_s: float = 2.0) -> None: + deadline = asyncio.get_event_loop().time() + timeout_s + while not predicate(): + if asyncio.get_event_loop().time() > deadline: + raise AssertionError("condition not reached") + await asyncio.sleep(0.01) + + +# --------------------------------------------------------------------------- +# Probe/benchmark helpers (ported from the retired MiniCPM demo client tests) + + +def test_build_realtime_url_with_model_extra_query(): + url = build_realtime_url( + "ws://localhost:8099/v1/realtime?custom=1", + "openbmb/MiniCPM-o-4_5", + session_id="session-a", + extra_query={"native_duplex": "1"}, + ) + + query = parse_qs(urlsplit(url).query) + assert query == { + "custom": ["1"], + "duplex": ["1"], + "model": ["openbmb/MiniCPM-o-4_5"], + "native_duplex": ["1"], + "session_id": ["session-a"], + } + + +def test_build_realtime_url_resume_only_when_autostart_disabled(): + url = build_realtime_url( + "ws://localhost:8099/v1/realtime?duplex=1", + "openbmb/MiniCPM-o-4_5", + autostart=False, + extra_query={"native_duplex": "1"}, + ) + + query = parse_qs(urlsplit(url).query) + assert query["autostart"] == ["0"] + assert query["native_duplex"] == ["1"] + + +def test_event_collector_partitions_audio_by_response(): + collector = EventCollector() + collector.add({"type": "response.created", "response": {"id": "resp-a"}}) + collector.add( + { + "type": "response.audio.delta", + "response_id": "resp-a", + "delta": base64.b64encode(b"audio-a").decode("ascii"), + "sample_rate_hz": 16_000, + } + ) + + assert collector.response_ids == ["resp-a"] + assert collector.audio_bytes("resp-a") == b"audio-a" + assert collector.output_sample_rate_hz == 16_000 + assert collector.first_received_at("response.created") is not None + assert collector.last_received_at("response.audio.delta") is not None + + +def test_event_collector_reports_engine_token_and_audio_intervals(): + collector = EventCollector() + collector.add( + {"type": "response.created", "response": {"id": "resp-a"}}, + received_at_s=10.0, + ) + stage_metrics = { + "0": { + "num_tokens_out": 4, + "vllm_ttft_ms": 120.0, + "vllm_tpot_ms": 15.0, + "vllm_itl_ms": 14.0, + "vllm_itls_ms": [10.0, 14.0, 18.0], + } + } + for received_at_s, cumulative_audio_ms in ((10.2, 80), (10.25, 160), (10.36, 240)): + collector.add( + { + "type": "response.audio.delta", + "response_id": "resp-a", + "delta": base64.b64encode(b"audio").decode("ascii"), + "sample_rate_hz": 16_000, + "metadata": { + "audio_duration_ms": cumulative_audio_ms, + "vllm_omni": {"stage_metrics": stage_metrics}, + }, + }, + received_at_s=received_at_s, + ) + collector.add( + {"type": "response.audio_transcript.delta", "response_id": "resp-a", "delta": ""}, + received_at_s=10.1, + ) + collector.add( + {"type": "response.audio_transcript.delta", "response_id": "resp-a", "delta": "hello"}, + received_at_s=10.15, + ) + collector.add( + {"type": "response.done", "response": {"id": "resp-a"}}, + received_at_s=10.4, + ) + + timing = collector.timing_summary( + after_s=10.0, + input_committed_at_s=9.9, + response_id="resp-a", + ) + + assert timing["stage0_tokens"] == { + "source": "engine_stage_metrics", + "output_token_count": 4, + "ttft_ms": 120.0, + "tpot_ms": 15.0, + "itls_ms": [10.0, 14.0, 18.0], + "inter_token_interval_ms": { + "count": 3, + "mean": 14.0, + "p50": 14.0, + "p95": 18.0, + "max": 18.0, + }, + } + assert timing["audio_output"] == { + "source": "client_monotonic_receive", + "chunk_count": 3, + "response_created_to_first_audio_ms": 200.0, + "commit_to_first_audio_ms": 300.0, + "inter_chunk_interval_ms": { + "count": 2, + "mean": 80.0, + "p50": 50.0, + "p95": 110.0, + "max": 110.0, + }, + "chunk_duration_ms": { + "count": 3, + "mean": 80.0, + "p50": 80.0, + "p95": 80.0, + "max": 80.0, + }, + "max_chunk_gap_ms": 110.0, + } + # Raw data only: derived metrics such as the RTF are the caller's job + # (e.g. vllm_omni.metrics.definitions.compute_audio_rtf). + assert timing["request_metrics"] == { + "source": "client_monotonic_receive", + "measurement_origin": { + "ttft": "input_audio_buffer.commit client send to first non-empty text delta", + "ttfp": "input_audio_buffer.commit client send to first audio packet", + }, + "ttft_ms": 250.0, + "ttfp_ms": 300.0, + "audio_generation_ms": 460.0, + "audio_duration_ms": 240.0, + } + + +def test_response_timing_ignores_unowned_session_level_metrics(): + collector = EventCollector() + collector.add( + {"type": "response.created", "response": {"id": "resp-a"}}, + received_at_s=10.0, + ) + collector.add( + { + "type": "response.audio.delta", + "response_id": "resp-a", + "delta": base64.b64encode(b"audio").decode("ascii"), + "metadata": { + "vllm_omni": { + "stage_metrics": { + "0": { + "num_tokens_out": 20, + "vllm_ttft_ms": 157.0, + "vllm_tpot_ms": 16.0, + "vllm_itls_ms": [15.0, 17.0], + } + } + } + }, + }, + received_at_s=10.2, + ) + collector.add( + { + "type": "response.listen", + "metadata": { + "vllm_omni": { + "stage_metrics": { + "0": { + "num_tokens_out": 2, + "vllm_ttft_ms": 106.0, + "vllm_tpot_ms": 0.0, + "vllm_itls_ms": [], + } + } + } + }, + }, + received_at_s=10.3, + ) + + timing = collector.timing_summary(after_s=10.0, response_id="resp-a") + + assert timing["stage0_tokens"]["output_token_count"] == 20 + assert timing["stage0_tokens"]["ttft_ms"] == 157.0 + + +def test_summarize_session_request_metrics_averages_audio_turns(): + summary = summarize_session_request_metrics( + [ + {"ttft_ms": 100.0, "ttfp_ms": 200.0, "rtf": 0.5}, + {"ttft_ms": 300.0, "ttfp_ms": 400.0, "rtf": 0.7}, + ], + session_id="sess-1", + ) + assert summary == { + "session_id": "sess-1", + "audio_turn_count": 2, + "mean_ttft_ms": 200.0, + "mean_ttfp_ms": 300.0, + "mean_rtf": 0.6, + } + + +def test_pcm16_wav_round_trip(tmp_path): + path = tmp_path / "audio.wav" + pcm16 = b"\x01\x00\x02\x00" + + write_pcm16_wav(path, pcm16, sample_rate_hz=16_000) + + with wave.open(str(path), "rb") as wav_file: + assert wav_file.getnchannels() == 1 + assert wav_file.getframerate() == 16_000 + assert read_pcm16_wav(path) == pcm16 + + +# --------------------------------------------------------------------------- +# Model-unit helpers and camera-frame interleaving + + +def test_duplex_unit_boundary_and_residual_math(): + assert duplex_unit_boundary_ms(0) == 1030 + assert duplex_unit_boundary_ms(2) == 3030 + assert has_residual_model_unit(b"\x00" * 32_000, chunk_period_ms=1000) is False + assert has_residual_model_unit(b"\x00" * 32_002, chunk_period_ms=1000) is True + created = {"type": "session.created", "session": {"capabilities": {"chunk_period_ms": 500}}} + assert chunk_period_ms([created]) == 500 + assert chunk_period_ms([{"type": "session.created", "session": {}}]) == 1000 + + +async def test_stream_pcm_sends_each_units_composite_beside_its_base_frame(): + sock = FakeSocket() + sock.feed(SESSION_CREATED) + client, _ = make_client(sock) + async with client: + frames_sent = await client.stream_pcm( + b"\x01\x00" * (16_000 * 3), + chunk_ms=200, + realtime=False, + video_frames=["f0", "f1"], + stacked_video_frames=["s0", None], + ) + appends = [event for event in sock.sent if event.get("type") == "input_audio_buffer.append"] + # A composite belongs to the unit it was captured in, so it rides the + # same append as that unit's base frame; a unit without one sends the + # base alone. Frame k rides the append that closes model unit k. + assert [event["video_frames"] for event in appends if "video_frames" in event] == [["f0", "s0"], ["f1"]] + assert [event["audio_end_ms"] for event in appends if "video_frames" in event] == [1200, 2200] + assert frames_sent == 2 + sock.feed(SESSION_CLOSED) + + +def test_build_realtime_url_native_duplex_flag_and_http_scheme(): + url = build_realtime_url("http://localhost:8099/v1/realtime", None, native_duplex=None) + parts = urlsplit(url) + assert parts.scheme == "ws" + assert parse_qs(parts.query) == {"duplex": ["1"]} + + url = build_realtime_url("https://host/v1/realtime", "m", native_duplex=False) + parts = urlsplit(url) + assert parts.scheme == "wss" + assert parse_qs(parts.query)["native_duplex"] == ["0"] + + with pytest.raises(ValueError): + build_realtime_url("ftp://host/v1/realtime", "m") + + +def test_reference_audio_data_url(tmp_path): + assert reference_audio_data_url(None) is None + path = tmp_path / "ref.wav" + path.write_bytes(b"RIFF") + assert reference_audio_data_url(str(path)) == "data:audio/wav;base64," + base64.b64encode(b"RIFF").decode("ascii") + with pytest.raises(FileNotFoundError): + reference_audio_data_url(str(tmp_path / "missing.wav")) + + +def test_event_collector_response_text_joins_deltas_per_response(): + collector = EventCollector() + collector.add({"type": "response.created", "response_id": "r1"}) + collector.add({"type": "response.audio_transcript.delta", "response_id": "r1", "delta": "he"}) + collector.add({"type": "response.output_text.delta", "response_id": "r2", "delta": "other"}) + collector.add({"type": "response.text.delta", "response_id": "r1", "delta": "llo"}) + assert collector.response_text("r1") == "hello" + assert collector.response_text("r2") == "other" + assert collector.response_text("r3") == "" diff --git a/tests/clients/test_model_session_configs.py b/tests/clients/test_model_session_configs.py new file mode 100644 index 000000000..00741d3c6 --- /dev/null +++ b/tests/clients/test_model_session_configs.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Model duplex session presets under ``vllm_omni.clients.``.""" + +import subprocess +import sys + +import pytest + +from vllm_omni.clients.duplex import AudioFormat +from vllm_omni.clients.minicpmo_4_5 import ( + create_duplex_session_config as create_minicpmo45_session_config, +) +from vllm_omni.clients.personaplex import ( + create_duplex_session_config as create_personaplex_session_config, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_minicpmo45_session_config_matches_native_duplex_deployment(): + config = create_minicpmo45_session_config(ref_audio="data:audio/wav;base64,AAA=", temperature=0.0) + payload = config.to_session_payload(model="openbmb/MiniCPM-o-4_5") + assert payload["ref_audio"] == "data:audio/wav;base64,AAA=" + assert payload["overlap_policy"] == "listen_only" + assert payload["playback_commit_policy"] == "ack_only" + assert payload["extra_body"]["native_duplex"] is True + assert payload["extra_body"]["auto_response"] is True + assert payload["temperature"] == 0.0 + + +def test_personaplex_session_config_matches_deployment(): + config = create_personaplex_session_config(voice="NATF2.pt", persona="You are calm.") + assert config.input_audio == AudioFormat("pcm_f32le", 24_000) + payload = config.to_session_payload(model="nvidia/personaplex-7b-v1") + assert payload["input_audio_format"] == "pcm_f32le" + assert payload["voice"] == "NATF2.pt" + assert payload["instructions"] == "You are calm." + + +@pytest.mark.parametrize( + "duplex_package", + [ + "vllm_omni.model_executor.models.minicpmo_4_5.duplex", + "vllm_omni.model_executor.models.personaplex.duplex", + ], +) +def test_model_duplex_packages_stay_clear_of_client_library(duplex_package: str): + """Model duplex plugin packages are server-side: no vllm_omni.clients.""" + script = f""" +import sys + +import {duplex_package} + +loaded = [ + name + for name in sys.modules + if name == "vllm_omni.clients" or name.startswith("vllm_omni.clients.") +] +if loaded: + raise SystemExit("model duplex package loaded the client library: " + ", ".join(loaded)) +""" + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/e2e/features/fullduplex/personaplex/test_import_boundary.py b/tests/e2e/features/fullduplex/personaplex/test_import_boundary.py deleted file mode 100644 index 463c0c98e..000000000 --- a/tests/e2e/features/fullduplex/personaplex/test_import_boundary.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import subprocess -import sys - - -def test_batched_runtime_import_does_not_require_optional_opus_dependency() -> None: - script = """ -import builtins - -real_import = builtins.__import__ - -def guarded_import(name, *args, **kwargs): - if name == "sphn": - raise ModuleNotFoundError("blocked optional dependency: sphn") - return real_import(name, *args, **kwargs) - -builtins.__import__ = guarded_import -from vllm_omni.experimental.fullduplex.personaplex.serving.batched import BatchedSessionManager -assert BatchedSessionManager is not None -""" - result = subprocess.run( - [sys.executable, "-c", script], - check=False, - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stderr diff --git a/tests/e2e/features/fullduplex/test_client.py b/tests/e2e/features/fullduplex/test_client.py index 53965b89e..266df1fa0 100644 --- a/tests/e2e/features/fullduplex/test_client.py +++ b/tests/e2e/features/fullduplex/test_client.py @@ -15,20 +15,20 @@ from vllm_omni.experimental.fullduplex.client import ( read_pcm16_wav, write_pcm16_wav, ) -from vllm_omni.experimental.fullduplex.minicpmo45.policy import ( - MiniCPMO45DuplexPolicy, -) from vllm_omni.experimental.fullduplex.video_stacking import ( concat_frames, unit_subframe_offsets, ) +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import ( + MiniCPMO45DuplexPolicy, +) pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def test_realtime_client_builds_explicit_native_duplex_url(): url = build_realtime_url( - "ws://localhost:8099/v1/realtime?custom=1&duplex=0&model=stale&minicpmo45_native_duplex=0&session_id=stale", + "ws://localhost:8099/v1/realtime?custom=1&duplex=0&model=stale&native_duplex=0&session_id=stale", "openbmb/MiniCPM-o-4_5", session_id="session-a", ) @@ -38,7 +38,7 @@ def test_realtime_client_builds_explicit_native_duplex_url(): "custom": ["1"], "duplex": ["1"], "model": ["openbmb/MiniCPM-o-4_5"], - "minicpmo45_native_duplex": ["1"], + "native_duplex": ["1"], "session_id": ["session-a"], } @@ -65,7 +65,7 @@ def test_realtime_client_builds_resume_only_url_when_autostart_disabled(): query = parse_qs(urlsplit(url).query) assert query["autostart"] == ["0"] - assert query["minicpmo45_native_duplex"] == ["1"] + assert query["native_duplex"] == ["1"] @pytest.mark.asyncio @@ -232,7 +232,7 @@ async def test_realtime_client_configure_explicit_tts_opts_out_of_native_duplex( session_extra_body = client.sent[0]["session"]["extra_body"] assert session_extra_body == { "ref_audio": "data:audio/wav;base64,AAAA", - "minicpmo45_native_duplex": False, + "native_duplex": False, } diff --git a/tests/e2e/features/fullduplex/test_core_contracts.py b/tests/e2e/features/fullduplex/test_core_contracts.py index 2a514b9f2..12a87640f 100644 --- a/tests/e2e/features/fullduplex/test_core_contracts.py +++ b/tests/e2e/features/fullduplex/test_core_contracts.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Contract tests for the fullduplex core (PR #3907 alignment). Covers the DuplexFence identity value, the session identity rules (epoch only @@ -14,6 +14,7 @@ import dataclasses import pytest +from vllm_omni.engine.duplex.messages import DuplexFence from vllm_omni.experimental.fullduplex.core import protocol as ev from vllm_omni.experimental.fullduplex.core.adapter import ( DuplexAdapter, @@ -26,7 +27,6 @@ from vllm_omni.experimental.fullduplex.core.session import ( DuplexSessionConfig, DuplexState, ) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/e2e/features/fullduplex/test_personaplex_adapter.py b/tests/e2e/features/fullduplex/test_personaplex_adapter.py deleted file mode 100644 index 458333627..000000000 --- a/tests/e2e/features/fullduplex/test_personaplex_adapter.py +++ /dev/null @@ -1,262 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""PersonaPlex adapter driven through the model-owned lockstep runtime. - -PersonaPlexDuplexRuntime starts ONE response for the whole session, streams one -agent frame per user frame, and drains on close (core stays verbatim upstream; -the lockstep lifecycle is model policy) — all with a GPU-free stub stepper. -""" - -import asyncio - -import numpy as np -import pytest - -from vllm_omni.experimental.fullduplex.core import protocol as ev -from vllm_omni.experimental.fullduplex.core.session import ( - DuplexSession, - DuplexSessionConfig, - DuplexState, -) -from vllm_omni.experimental.fullduplex.personaplex.adapter import ( - PersonaPlexDuplexAdapter, - PersonaPlexDuplexRuntime, -) -from vllm_omni.experimental.fullduplex.personaplex.config import FRAME_SIZE -from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -class StubStepper: - """Deterministic FrameStepper: frame i -> audio filled with i, text 't{i}' on odds.""" - - sample_rate = 24000 - frame_size = FRAME_SIZE - - def __init__(self) -> None: - self.opened: tuple[str | None, str | None] | None = None - self.steps = 0 - - def open_session(self, voice_prompt=None, persona=None) -> None: - self.opened = (voice_prompt, persona) - self.steps = 0 - - def step(self, user_pcm): - assert user_pcm.shape[0] == self.frame_size - self.steps += 1 - i = self.steps - return FrameOutput( - audio=np.full(self.frame_size, float(i), dtype=np.float32), - text=(f"t{i}" if i % 2 else None), - ) - - -class RecordingAdapter(PersonaPlexDuplexAdapter): - def __init__(self, stepper: StubStepper) -> None: - super().__init__(stepper) - self.closed = False - - async def on_close(self, session: DuplexSession) -> None: - self.closed = True - await super().on_close(session) - - -def _collector(): - out: list[dict] = [] - - async def emit(event: dict) -> None: - out.append(event) - - return out, emit - - -async def _feed(events): - for e in events: - yield e - - -async def _raising_feed(frame): - yield {"type": ev.INPUT_APPEND, "modality": "audio", "data": frame} - await asyncio.sleep(0) - raise RuntimeError("input failed") - - -def _respond_tasks() -> list[asyncio.Task]: - current = asyncio.current_task() - return [ - task - for task in asyncio.all_tasks() - if task is not current and "_respond_once" in getattr(task.get_coro(), "__qualname__", "") - ] - - -def _session(): - cfg = DuplexSessionConfig( - input_modalities=("audio",), - output_modalities=("audio", "text"), - ) - return DuplexSession("s", cfg) - - -@pytest.mark.asyncio -async def test_lockstep_one_agent_frame_per_user_frame(): - stub = StubStepper() - adapter = PersonaPlexDuplexAdapter(stub, voice_prompt="NATF2.pt", persona="be terse") - rt = PersonaPlexDuplexRuntime(_session(), adapter) - out, emit = _collector() - - frame = np.zeros(FRAME_SIZE, dtype=np.float32) - events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame} for _ in range(3)] - events.append({"type": ev.CLOSE}) - await rt.run(_feed(events), emit) - - types = [e["type"] for e in out] - # exactly one response for the whole session, drained cleanly on close - assert types.count(ev.RESPONSE_CREATED) == 1 - assert types.count(ev.RESPONSE_DONE) == 1 - audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"] - text = [e["data"] for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "text"] - assert len(audio) == 3 # one agent frame per user frame - assert [int(e["data"][0]) for e in audio] == [1, 2, 3] # lockstep order - assert text == ["t1", "t3"] - assert stub.steps == 3 - assert stub.opened == ("NATF2.pt", "be terse") - - -@pytest.mark.asyncio -async def test_unaligned_chunks_are_reframed_to_80ms(): - stub = StubStepper() - adapter = PersonaPlexDuplexAdapter(stub) - rt = PersonaPlexDuplexRuntime(_session(), adapter) - out, emit = _collector() - - # 2.5 frames in one chunk + 0.5 frame in another => 3 whole frames - events = [ - {"type": ev.INPUT_APPEND, "modality": "audio", "data": np.zeros(FRAME_SIZE * 5 // 2, dtype=np.float32)}, - {"type": ev.INPUT_APPEND, "modality": "audio", "data": np.zeros(FRAME_SIZE // 2, dtype=np.float32)}, - {"type": ev.CLOSE}, - ] - await rt.run(_feed(events), emit) - audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"] - assert len(audio) == 3 - assert stub.steps == 3 - - -@pytest.mark.asyncio -async def test_close_without_input_is_clean(): - adapter = PersonaPlexDuplexAdapter(StubStepper()) - rt = PersonaPlexDuplexRuntime(_session(), adapter) - out, emit = _collector() - await rt.run(_feed([{"type": ev.CLOSE}]), emit) - # no input -> response never started -> no created/done, no error - assert not any(e["type"] == ev.ERROR for e in out) - assert not any(e["type"] == ev.RESPONSE_CREATED for e in out) - - -@pytest.mark.asyncio -async def test_iterator_eof_without_close_drains_and_stops(): - # Transport drop: the input iterator ends WITHOUT an explicit CLOSE event. - # The runtime must still signal on_close so the eternal response drains - # instead of blocking on its inbox forever. - stub = StubStepper() - adapter = PersonaPlexDuplexAdapter(stub) - rt = PersonaPlexDuplexRuntime(_session(), adapter) - out, emit = _collector() - frame = np.zeros(FRAME_SIZE, dtype=np.float32) - events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}] - await asyncio.wait_for(rt.run(_feed(events), emit), timeout=5.0) - types = [e["type"] for e in out] - assert types.count(ev.RESPONSE_DONE) == 1 - assert stub.steps == 1 - - -@pytest.mark.asyncio -async def test_input_exception_closes_response_and_runtime() -> None: - stub = StubStepper() - adapter = RecordingAdapter(stub) - rt = PersonaPlexDuplexRuntime(_session(), adapter) - out, emit = _collector() - frame = np.zeros(FRAME_SIZE, dtype=np.float32) - - with pytest.raises(RuntimeError, match="input failed"): - await rt.run(_raising_feed(frame), emit) - - leaked = _respond_tasks() - for task in leaked: - task.cancel() - if leaked: - await asyncio.gather(*leaked, return_exceptions=True) - - assert adapter.closed is True - assert rt.session.state is DuplexState.CLOSED - assert not leaked - - -@pytest.mark.asyncio -async def test_default_session_config_gets_lockstep_lifecycle(): - # The lifecycle comes from the model-owned runtime class, not a session - # flag: a plain default-constructed session config runs lockstep. - stub = StubStepper() - adapter = PersonaPlexDuplexAdapter(stub) - session = DuplexSession( - "s", - DuplexSessionConfig(input_modalities=("audio",), output_modalities=("audio", "text")), - ) - rt = PersonaPlexDuplexRuntime(session, adapter) - out, emit = _collector() - frame = np.zeros(FRAME_SIZE, dtype=np.float32) - events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}, {"type": ev.CLOSE}] - await rt.run(_feed(events), emit) - types = [e["type"] for e in out] - assert types.count(ev.RESPONSE_CREATED) == 1 - assert types.count(ev.RESPONSE_DONE) == 1 - assert stub.steps == 1 - - -@pytest.mark.asyncio -async def test_partial_tail_is_padded_and_stepped_on_close(): - # A trailing sub-frame chunk is zero-padded and stepped at close (same - # contract as PersonaPlexSession.flush) rather than silently dropped. - stub = StubStepper() - adapter = PersonaPlexDuplexAdapter(stub) - rt = PersonaPlexDuplexRuntime(_session(), adapter) - out, emit = _collector() - events = [ - {"type": ev.INPUT_APPEND, "modality": "audio", "data": np.ones(FRAME_SIZE // 2, dtype=np.float32)}, - {"type": ev.CLOSE}, - ] - await rt.run(_feed(events), emit) - audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"] - assert len(audio) == 1 - assert stub.steps == 1 - - -@pytest.mark.asyncio -async def test_single_eternal_response_across_barge_in(): - # Moved from the core contract tests when the lockstep lifecycle became - # model-owned: ONE response for the whole session, RESPONSE_CANCEL bumps the - # epoch fence (playback cursor) but the stream keeps flowing, and close - # drains via on_close. - stub = StubStepper() - adapter = PersonaPlexDuplexAdapter(stub) - session = _session() - rt = PersonaPlexDuplexRuntime(session, adapter) - out, emit = _collector() - - frame = np.zeros(FRAME_SIZE, dtype=np.float32) - events = [ - {"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}, - {"type": ev.RESPONSE_CANCEL}, # barge-in: epoch bumps, stream continues - {"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}, - {"type": ev.CLOSE}, - ] - await rt.run(_feed(events), emit) - - types = [e["type"] for e in out] - assert types.count(ev.RESPONSE_CREATED) == 1, "exactly one eternal response" - assert types.count(ev.RESPONSE_DONE) == 1 - audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"] - assert len(audio) == 2, "stream keeps flowing across barge-in" - assert session.epoch == 1, "cancel still advanced the epoch fence" - assert stub.steps == 2 diff --git a/tests/e2e/features/fullduplex/test_personaplex_batched.py b/tests/e2e/features/fullduplex/test_personaplex_batched.py deleted file mode 100644 index f2ee73729..000000000 --- a/tests/e2e/features/fullduplex/test_personaplex_batched.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""GPU-free tests for the batched PersonaPlex slot manager (stub engine).""" - -from types import SimpleNamespace - -import numpy as np -import pytest - -from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput, PrefillStep -from vllm_omni.experimental.fullduplex.personaplex.serving.batched import ( - BatchedSessionManager, - SlotPhase, -) - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - -FRAME = 4 - - -class StubEngine: - """Records the slot-API call sequence; echoes each row's pcm as its output.""" - - frame_size = FRAME - - def __init__(self, batch_size: int = 4, prefill_len: int = 2) -> None: - self.config = SimpleNamespace(batch_size=batch_size) - self.prefill_len = prefill_len - self.recycled: list[int] = [] - self.voice_ended: list[int] = [] - self.prefill_done: list[int] = [] - self.steps: list[tuple[np.ndarray, dict | None]] = [] - - def prefill_steps(self, persona=None): - steps = [PrefillStep("sacrifice"), PrefillStep("voice", embedding=None), PrefillStep("voice_end")] - steps += [PrefillStep("tokens", moshi_tokens=None, text_token=3, user_sine=True)] * self.prefill_len - return steps - - def recycle_slot(self, b): - self.recycled.append(b) - - def voice_end(self, b): - self.voice_ended.append(b) - - def end_prefill(self, b): - self.prefill_done.append(b) - - def step_batch(self, pcm, prefill=None): - self.steps.append((pcm.copy(), dict(prefill) if prefill else None)) - return [FrameOutput(audio=pcm[b].copy(), text=None) for b in range(self.config.batch_size)] - - -def test_all_idle_does_not_step(): - eng = StubEngine() - mgr = BatchedSessionManager(eng) - assert mgr.tick() is False - assert eng.steps == [] - - -def test_pristine_slot_is_live_immediately(): - eng = StubEngine() - mgr = BatchedSessionManager(eng) - slot, ready_now = mgr.acquire() - assert ready_now is True - assert slot.phase is SlotPhase.LIVE - assert eng.recycled == [] - - -def test_custom_persona_prefills_and_recycles_on_tick_thread(): - eng = StubEngine(prefill_len=2) - mgr = BatchedSessionManager(eng) - slot, ready_now = mgr.acquire(persona="custom") - assert ready_now is False - assert slot.phase is SlotPhase.PREFILL - assert eng.recycled == [] # deferred: reset must run between engine steps - - ready = [] - slot.on_ready = lambda: ready.append(True) - - mgr.tick() # recycle + sacrifice - assert eng.recycled == [slot.index] - assert eng.steps[-1][1][slot.index].kind == "sacrifice" - - mgr.tick() # voice frame - assert eng.steps[-1][1][slot.index].kind == "voice" - assert eng.voice_ended == [] - - mgr.tick() # voice_end executed BETWEEN ticks, then first tokens frame - assert eng.voice_ended == [slot.index] - assert eng.steps[-1][1][slot.index].kind == "tokens" - - mgr.tick() # second tokens frame - assert not ready - mgr.tick() # prefill exhausted -> live - assert eng.prefill_done == [slot.index] - assert slot.phase is SlotPhase.LIVE - assert ready == [True] - - -def test_pristine_slot_after_ticking_requires_recycle(): - # step_batch advances EVERY row each tick, so a pristine idle row that has - # lived through ticks has drifted from its boot prefill and must be - # recycled before a client may join it. - eng = StubEngine() - mgr = BatchedSessionManager(eng) - first, ready_now = mgr.acquire() - assert ready_now is True # claimed before any tick: truly pristine - mgr.tick() - second, ready_now2 = mgr.acquire() - assert second.index != first.index - assert ready_now2 is False - assert second.phase is SlotPhase.PREFILL - - -def test_used_slot_requires_recycle_even_with_default_persona(): - eng = StubEngine() - mgr = BatchedSessionManager(eng) - slot, ready_now = mgr.acquire() - assert ready_now - mgr.release(slot) - slot2, ready_now2 = mgr.acquire() - assert slot2.index == slot.index - assert ready_now2 is False - assert slot2.phase is SlotPhase.PREFILL - - -def test_acquire_returns_none_when_full(): - eng = StubEngine(batch_size=2) - mgr = BatchedSessionManager(eng) - assert mgr.acquire() is not None - assert mgr.acquire() is not None - assert mgr.acquire() is None - - -def test_feed_frames_epoch_guard_and_backlog_cap(): - eng = StubEngine() - mgr = BatchedSessionManager(eng, max_buffered_frames=3) - slot, _ = mgr.acquire() - mgr.feed(slot, np.arange(FRAME * 2, dtype=np.float32), slot.epoch) - assert len(slot.inq) == 2 - mgr.feed(slot, np.zeros(3, dtype=np.float32), slot.epoch) # partial tail buffered - assert len(slot.inq) == 2 and slot.pending.shape[0] == 3 - mgr.feed(slot, np.zeros(FRAME * 5, dtype=np.float32), slot.epoch) - assert len(slot.inq) == 3 # oldest dropped beyond the cap - mgr.feed(slot, np.zeros(FRAME, dtype=np.float32), slot.epoch - 1) # stale epoch ignored - assert len(slot.inq) == 3 - - -def test_live_slot_consumes_input_and_emits_output(): - eng = StubEngine() - mgr = BatchedSessionManager(eng) - slot, _ = mgr.acquire() - got = [] - slot.on_output = got.append - frame = np.full(FRAME, 0.5, dtype=np.float32) - mgr.feed(slot, frame, slot.epoch) - mgr.tick() - assert len(got) == 1 - np.testing.assert_array_equal(got[0].audio, frame) - # queue drained: next tick feeds zeros - mgr.tick() - np.testing.assert_array_equal(got[1].audio, np.zeros(FRAME, dtype=np.float32)) - - -def test_release_mid_flight_stops_emission(): - eng = StubEngine() - mgr = BatchedSessionManager(eng) - slot, _ = mgr.acquire() - got = [] - slot.on_output = got.append - mgr.tick() - assert len(got) == 1 - mgr.release(slot) - # another slot keeps the batch non-idle - other, _ = mgr.acquire() # same slot index, now PREFILL (used) - assert other.phase is SlotPhase.PREFILL - mgr.tick() - assert len(got) == 1 # no further output for the released epoch diff --git a/tests/e2e/features/fullduplex/test_personaplex_elastic.py b/tests/e2e/features/fullduplex/test_personaplex_elastic.py deleted file mode 100644 index 1b2f47984..000000000 --- a/tests/e2e/features/fullduplex/test_personaplex_elastic.py +++ /dev/null @@ -1,245 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project -"""GPU e2e for PersonaPlex elastic batching (bit-exact isolation + recycle). - -Greedy decoding makes every cross-slot leak observable as a bit difference: - -1. Isolation (bit-exact): recycling one slot mid-flight leaves every other - slot's output bit-identical to a run that never recycled. -2. Replay exactness: a recycled slot's opening live ticks match a pristine boot - at numerical-noise level (a broken KV mask / staging roll garbles frame 0). -3. Coherence (behavioral): a recycled slot fed a real spoken question produces - a real reply in its inner-monologue text stream. - -Run (moshi-free native stack; ~30 GB GPU + HF access to the gated repo; set -PPLEX_QUESTION_WAV to a spoken-question wav to enable the coherence test): - pytest tests/e2e/features/fullduplex/test_personaplex_elastic.py -v -s - -Bench (B sweep, ms/tick vs the 80 ms budget): - python tests/e2e/features/fullduplex/test_personaplex_elastic.py --bench 2 4 8 16 -""" - -from __future__ import annotations - -import numpy as np -import pytest - -from tests.helpers.mark import hardware_marks - -torch = pytest.importorskip("torch") - -# slow + H100 + omni + cards_1 routes this module into weekly -# "Omni · H100 · Single-GPU" (test-weekly.yml). -pytestmark = [ - pytest.mark.omni, - pytest.mark.slow, - *hardware_marks(res={"cuda": "H100"}), -] - -if not torch.cuda.is_available(): # noqa: E402 - pytest.skip("needs CUDA", allow_module_level=True) - -from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig # noqa: E402 -from vllm_omni.experimental.fullduplex.personaplex.runtime import ( # noqa: E402 - PersonaPlexEngine, -) - -B = 4 -LIVE_TICKS = 40 # live frames compared per claim -RECYCLE_AT = 10 - - -def _make_engine() -> PersonaPlexEngine: - cfg = PersonaPlexConfig(batch_size=B) # native stepper is greedy-only - try: - return PersonaPlexEngine(cfg).load() - except Exception as exc: # gated-repo access is environment-dependent - name = type(exc).__name__ - if "Gated" in name or "401" in str(exc) or "403" in str(exc): - pytest.skip(f"no access to the gated nvidia/personaplex-7b-v1 repo: {name}") - raise - - -@pytest.fixture(scope="module") -def engine() -> PersonaPlexEngine: - return _make_engine() - - -def _slot_stream(slot: int, tick: int, frame: int) -> np.ndarray: - """Deterministic per-slot input: distinct low-amplitude tones.""" - sr = 24000 - freq = 180.0 + 90.0 * slot - t0 = tick * frame - t = (np.arange(frame) + t0) / sr - return (0.12 * np.sin(2 * np.pi * freq * t)).astype(np.float32) - - -def _run(engine, total_ticks: int, recycle: bool): - """Run the lockstep loop; optionally recycle slot 1 at RECYCLE_AT. - - Returns per-slot lists of (audio | None) per live tick index, where the - recycled slot's list restarts its input stream at its post-prefill tick. - """ - frame = engine.frame_size - outs: list[list[np.ndarray | None]] = [[] for _ in range(B)] - stream_tick = [0] * B # per-slot conversation clock driving its input - prefill = None - tick = 0 - while tick < total_ticks: - if recycle and tick == RECYCLE_AT and prefill is None: - engine.recycle_slot(1) - prefill = list(engine.prefill_steps(None)) - pcm = np.zeros((B, frame), dtype=np.float32) - step_for_1 = None - if prefill: - while prefill and prefill[0].kind == "voice_end": - engine.voice_end(1) - prefill.pop(0) - if prefill: - step_for_1 = prefill.pop(0) - else: - engine.end_prefill(1) - prefill = None - stream_tick[1] = 0 # restart slot 1's input stream from its own t=0 - for b in range(B): - if b == 1 and step_for_1 is not None: - continue # engine overrides the user channel for prefill rows - pcm[b] = _slot_stream(b, stream_tick[b], frame) - stream_tick[b] += 1 - frame_outs = engine.step_batch(pcm, {1: step_for_1} if step_for_1 is not None else None) - for b in range(B): - if b == 1 and step_for_1 is not None: - continue # prefill tick: nothing conversational to record - frame_out = frame_outs[b] - outs[b].append(None if frame_out is None else frame_out.audio) - tick += 1 - return outs - - -def test_recycle_isolation_and_fresh_equivalence(engine): - engine.open_batch() - prefill_len = sum(1 for s in engine.prefill_steps(None) if s.kind != "voice_end") - total = RECYCLE_AT + prefill_len + LIVE_TICKS - - base = _run(engine, total, recycle=False) - - engine.open_batch() # deterministic re-boot (greedy, fixed voice/persona) - rec = _run(engine, total, recycle=True) - - # Claim 1 -- isolation: untouched slots bit-identical to the no-recycle run. - for b in (0, 2, 3): - assert len(base[b]) == total and len(rec[b]) == total - for i, (x, y) in enumerate(zip(base[b], rec[b])): - assert (x is None) == (y is None), f"slot {b} tick {i} validity diverged" - if x is not None: - assert np.array_equal(x, y), f"slot {b} tick {i} audio diverged (recycle leaked)" - - # Claim 2 -- replay exactness: the recycled slot's opening live ticks match a - # pristine boot at numerical-noise level. A broken KV mask / staging-ring - # roll / Mimi reset garbles the audio from the very first frame (diffs of - # 0.1-1.0); a correct replay differs only by decoder-state rounding (~1e-4). - # - # Full-length bitwise identity is NOT expected, for two structural reasons: - # the replayed system prompt sits at a different ABSOLUTE RoPE offset (bf16 - # rounds differently there), and the recycled Mimi ENCODER cannot restore a - # fresh stream's start padding (a batch-shared scalar), so from the first - # non-silent user frame it emits different-but-equally-valid codes for the - # same audio. Once any near-tie flips a token, the two equally valid - # trajectories legitimately diverge; coherence of the recycled slot is - # asserted behaviorally in test_recycled_slot_converses. - fresh = [a for a in base[1] if a is not None][:LIVE_TICKS] - reborn = [a for a in rec[1][RECYCLE_AT:] if a is not None][:LIVE_TICKS] - assert len(reborn) >= LIVE_TICKS // 2, "recycled slot produced too few live frames" - diffs = [float(np.max(np.abs(x - y))) for x, y in zip(fresh, reborn)] - first_flip = next((i for i, d in enumerate(diffs) if d > 5e-2), len(diffs)) - print( - f"[elastic] replay exactness: {first_flip}/{len(diffs)} opening ticks at noise level " - f"(head diffs: {[f'{d:.1e}' for d in diffs[:6]]})" - ) - assert first_flip >= 4, ( - f"recycled slot diverged from fresh behavior at tick {first_flip} " - f"(diffs {diffs[: first_flip + 2]}); the system-prompt replay is broken" - ) - assert max(diffs[:4]) < 2e-2, f"reborn audio not at noise level: {max(diffs[:4])}" - - -def test_recycled_slot_converses(engine): - """Behavioral guarantee: a recycled slot holds a coherent conversation. - - Feed a real spoken question into a slot that was recycled mid-flight (while - other slots keep running); the model's own inner-monologue text stream must - produce a real reply. This is the end-user property the encoder-state - caveats above cannot break. - """ - import os - - import sphn - - wav_path = os.environ.get("PPLEX_QUESTION_WAV") - if not wav_path: - pytest.skip("set PPLEX_QUESTION_WAV to a spoken-question wav to run") - wav, _ = sphn.read(wav_path, sample_rate=24000) - wav = wav[0].astype(np.float32) - frame = engine.frame_size - pad = (-len(wav)) % frame - wav = np.concatenate([wav, np.zeros(pad, dtype=np.float32)]) - q_frames = [wav[i : i + frame] for i in range(0, len(wav), frame)] - - engine.open_batch() - pcm = np.zeros((B, frame), dtype=np.float32) - for _ in range(5): # everyone live briefly - engine.step_batch(pcm) - engine.recycle_slot(1) - prefill = list(engine.prefill_steps(None)) - while prefill: - while prefill and prefill[0].kind == "voice_end": - engine.voice_end(1) - prefill.pop(0) - if not prefill: - break - engine.step_batch(pcm, {1: prefill.pop(0)}) - engine.end_prefill(1) - - text = "" - total = len(q_frames) + 120 # question + ~10 s to answer - for i in range(total): - pcm = np.zeros((B, frame), dtype=np.float32) - if i < len(q_frames): - pcm[1] = q_frames[i] - outs = engine.step_batch(pcm) - if outs[1] is not None and outs[1].text: - text += outs[1].text - alpha = sum(c.isalpha() for c in text) - print(f"[elastic] recycled slot inner monologue: {text!r}") - assert alpha >= 15, f"recycled slot produced no coherent reply: {text!r}" - - -def _bench(batch_sizes: list[int]) -> None: - import time - - for bs in batch_sizes: - cfg = PersonaPlexConfig(batch_size=max(bs, 2)) - eng = PersonaPlexEngine(cfg).load() - eng.open_batch() - frame = eng.frame_size - pcm = np.zeros((cfg.batch_size, frame), dtype=np.float32) - for _ in range(10): # warm - eng.step_batch(pcm) - torch.cuda.synchronize() - t0 = time.monotonic() - n = 100 - for _ in range(n): - eng.step_batch(pcm) - torch.cuda.synchronize() - ms = (time.monotonic() - t0) / n * 1000 - mem = torch.cuda.max_memory_allocated() / 2**30 - print(f"B={cfg.batch_size:3d} {ms:6.1f} ms/tick RTF/stream {ms / 80:.3f} {mem:.1f} GiB") - del eng - torch.cuda.empty_cache() - - -if __name__ == "__main__": - import sys - - sizes = [int(x) for x in sys.argv[2:]] if len(sys.argv) > 2 and sys.argv[1] == "--bench" else [2, 4, 8] - _bench(sizes) diff --git a/tests/e2e/features/fullduplex/test_personaplex_server_lease.py b/tests/e2e/features/fullduplex/test_personaplex_server_lease.py deleted file mode 100644 index 451ffb714..000000000 --- a/tests/e2e/features/fullduplex/test_personaplex_server_lease.py +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Single-session lease teardown: a cancelled engine call must be drained. - -Cancelling an asyncio task blocked on a worker-thread engine call returns -immediately while the thread keeps running (executor futures cannot be -interrupted). The server therefore tracks the in-flight future on the session -state and waits it out before releasing the capacity-one lease; otherwise a -fast reconnect could call ``open()`` on the shared engine while the previous -connection's final ``step()`` is still mutating it. -""" - -import asyncio -import contextlib -import sys -import threading -import types - -import pytest - -try: - import sphn # noqa: F401 -except ImportError: - # These lease tests never encode/decode Opus. Keep the optional serving - # dependency from hiding dependency-free teardown regressions. - sys.modules["sphn"] = types.ModuleType("sphn") - -from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig -from vllm_omni.experimental.fullduplex.personaplex.serving.server import DuplexServer -from vllm_omni.experimental.fullduplex.personaplex.session import PersonaPlexServingSessionState - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -@pytest.mark.asyncio -async def test_cancelled_engine_call_is_drained_before_lease_release(): - server = DuplexServer(PersonaPlexConfig()) # engine never loaded: stub calls only - state = PersonaPlexServingSessionState() - - started = threading.Event() - release = threading.Event() - finished = threading.Event() - - def slow_engine_op(): - started.set() - release.wait(timeout=5.0) - finished.set() - return [] - - task = asyncio.create_task(server._engine_call(state, slow_engine_op)) - await asyncio.to_thread(started.wait, 5.0) - - # Cancel while the worker thread is mid-call: the await returns at once, - # the thread keeps running, and its completion event stays tracked on the - # state (the asyncio future itself reads as cancelled/done here, which is - # exactly why the drain must not rely on it). - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - assert state.inflight is not None - assert not state.inflight.is_set() - - # The drain must genuinely wait for the worker, not just return. - drain = asyncio.create_task(server._drain_inflight(state)) - await asyncio.sleep(0.05) - assert not drain.done() - - release.set() - await asyncio.wait_for(drain, timeout=5.0) - assert finished.is_set() - assert state.inflight.is_set() - - -@pytest.mark.asyncio -async def test_completed_engine_call_clears_inflight(): - server = DuplexServer(PersonaPlexConfig()) - state = PersonaPlexServingSessionState() - result = await server._engine_call(state, lambda: "ok") - assert result == "ok" - assert state.inflight is None - await server._drain_inflight(state) # no-op on a clean state - - -@pytest.mark.asyncio -async def test_drain_timeout_keeps_lease_until_worker_finishes(): - server = DuplexServer(PersonaPlexConfig()) - server._active = True - state = PersonaPlexServingSessionState() - state.inflight = threading.Event() - - drained = await server._release_lease_after_drain(state, timeout=0.01) - - assert drained is False - assert server._active is True - - state.inflight.set() - for _ in range(100): - if not server._active: - break - await asyncio.sleep(0.01) - assert server._active is False diff --git a/tests/e2e/features/fullduplex/test_personaplex_session.py b/tests/e2e/features/fullduplex/test_personaplex_session.py deleted file mode 100644 index f838de4ff..000000000 --- a/tests/e2e/features/fullduplex/test_personaplex_session.py +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Lockstep-contract tests for the PersonaPlex session driver (no GPU / no moshi). - -A stub :class:`FrameStepper` stands in for the real Moshi backend so we can assert -the framing + 1-frame-in / 1-frame-out invariants without weights. -""" - -import numpy as np -import pytest - -from vllm_omni.experimental.fullduplex.personaplex.config import FRAME_SIZE, PersonaPlexConfig -from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput, FrameStepper -from vllm_omni.experimental.fullduplex.personaplex.session import PersonaPlexSession - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -class StubStepper: - """Deterministic FrameStepper: frame i -> audio filled with i, text 't{i}' on odds.""" - - sample_rate = 24000 - frame_size = FRAME_SIZE - - def __init__(self) -> None: - self.opened: tuple[str | None, str | None] | None = None - self.steps = 0 - - def open_session(self, voice_prompt=None, persona=None) -> None: - self.opened = (voice_prompt, persona) - self.steps = 0 - - def step(self, user_pcm): - assert user_pcm.shape[0] == self.frame_size - self.steps += 1 - i = self.steps - return FrameOutput( - audio=np.full(self.frame_size, float(i), dtype=np.float32), - text=(f"t{i}" if i % 2 else None), - ) - - -def test_stub_satisfies_protocol(): - assert isinstance(StubStepper(), FrameStepper) - - -def test_feed_emits_one_output_per_complete_frame(): - s = PersonaPlexSession(StubStepper(), PersonaPlexConfig()) - s.open(voice_prompt="NATF2.pt", persona="be terse") - out = s.feed(np.zeros(FRAME_SIZE * 3, dtype=np.float32)) - assert len(out) == 3 - # lockstep order preserved: frame i -> audio filled with i - assert [int(o.audio[0]) for o in out] == [1, 2, 3] - assert [o.text for o in out] == ["t1", None, "t3"] - - -def test_partial_frames_are_buffered_until_complete(): - stub = StubStepper() - s = PersonaPlexSession(stub, PersonaPlexConfig()) - s.open() - assert s.feed(np.zeros(FRAME_SIZE // 2, dtype=np.float32)) == [] # nothing yet - assert stub.steps == 0 - out = s.feed(np.zeros(FRAME_SIZE, dtype=np.float32)) # 1.5 frames total -> 1 frame - assert len(out) == 1 - assert stub.steps == 1 - - -def test_flush_pads_the_tail(): - stub = StubStepper() - s = PersonaPlexSession(stub, PersonaPlexConfig()) - s.open() - s.feed(np.ones(FRAME_SIZE + 10, dtype=np.float32)) # 1 frame + 10-sample tail - assert stub.steps == 1 - tail = s.flush() - assert len(tail) == 1 and stub.steps == 2 - assert s.flush() == [] # nothing left - - -def test_run_concatenates_audio_and_joins_text(): - s = PersonaPlexSession(StubStepper(), PersonaPlexConfig()) - s.open() - audio, text = s.run(np.zeros(FRAME_SIZE * 4, dtype=np.float32)) - assert audio.shape[0] == FRAME_SIZE * 4 - assert text == "t1t3" # frames 1..4, text on odds - - -def test_open_forwards_voice_and_persona(): - stub = StubStepper() - s = PersonaPlexSession(stub, PersonaPlexConfig()) - s.open(voice_prompt="NATM1.pt", persona="pirate") - assert stub.opened == ("NATM1.pt", "pirate") - - -def test_feed_before_open_raises(): - s = PersonaPlexSession(StubStepper(), PersonaPlexConfig()) - with pytest.raises(RuntimeError): - s.feed(np.zeros(FRAME_SIZE, dtype=np.float32)) diff --git a/tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py b/tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py index 01afe689a..432124e76 100644 --- a/tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py +++ b/tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py @@ -15,11 +15,26 @@ import pytest from huggingface_hub import snapshot_download from tests.helpers.runtime import OmniServerParams, get_model_prefix -from tests.helpers.stage_config import get_deploy_config_path, get_deploy_duplex_max_sessions +from tests.helpers.stage_config import ( + get_deploy_config_path, + get_deploy_duplex_max_sessions, + modify_stage_config, +) MODEL = "openbmb/MiniCPM-o-4_5" DEPLOY_CONFIG_REL = "minicpmo_4_5.yaml" DEPLOY_CONFIG = get_deploy_config_path(DEPLOY_CONFIG_REL) +# Eager-execution variant for fast-startup core-tier probes (e.g. the duplex +# client live test): skips CUDA-graph capture on the LLM and Talker stages. +CORE_DEPLOY_CONFIG = modify_stage_config( + DEPLOY_CONFIG, + updates={ + "stages": { + 0: {"enforce_eager": True}, + 1: {"enforce_eager": True}, + } + }, +) ASSET_DIR = Path(__file__).resolve().parents[3] / "assets" / "minicpmo_4_5" RESPONSE_REQUIRED_WAV = ASSET_DIR / "response_required_16k.wav" RESPONSE_REQUIRED_SHA256 = "2e5fd4eb3ee434ce107ee3a0591fa624a33f7683c7462f45fe651c443c9af941" @@ -39,6 +54,18 @@ SERVER_PARAMS = [ ) ] +CORE_SERVER_PARAMS = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=CORE_DEPLOY_CONFIG, + use_stage_cli=False, + server_args=["--trust-remote-code"], + ), + id="three-stage-single-gpu", + ) +] + def deploy_max_sessions() -> int: """Concurrent duplex sessions the deploy config under test admits. diff --git a/tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py b/tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py index 7776f2a0f..ebd59fc3e 100644 --- a/tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py +++ b/tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py @@ -52,28 +52,49 @@ try: except ImportError as exc: # pragma: no cover - demo dependency. raise SystemExit("Install websockets first: pip install websockets") from exc -from vllm_omni.experimental.fullduplex.client import ( # noqa: E402 +from vllm_omni.clients.duplex import ( # noqa: E402 PCM16_BYTES_PER_SAMPLE, PCM16_SAMPLE_RATE, - RealtimeEventCollector, + EventCollector, build_realtime_url, duplex_unit_boundary_ms, read_pcm16_wav, summarize_session_request_metrics, ) -from vllm_omni.experimental.fullduplex.client import ( # noqa: E402 +from vllm_omni.clients.duplex import ( # noqa: E402 reference_audio_data_url as _ref_audio_data_url, ) from vllm_omni.experimental.fullduplex.video_stacking import ( # noqa: E402 concat_frames_b64, unit_subframe_offsets, ) +from vllm_omni.metrics.definitions import compute_audio_rtf # noqa: E402 + + +def _url_with_model(url, model, *, autostart=None, session_id=None): + # This driver speaks the MiniCPM-o native duplex wire contract. + return build_realtime_url( + url, + model, + autostart=autostart, + session_id=session_id, + extra_query={"native_duplex": "1"}, + ) + -_url_with_model = build_realtime_url _read_wav_pcm16 = read_pcm16_wav DemoArgs = argparse.Namespace | SimpleNamespace +def _audio_rtf_from_raw_metric(raw_metric: dict[str, object]) -> float | None: + """Derive audio RTF from the raw timings reported by the client.""" + generation_ms = raw_metric.get("audio_generation_ms") + duration_ms = raw_metric.get("audio_duration_ms") + if not isinstance(generation_ms, int | float) or not isinstance(duration_ms, int | float) or duration_ms <= 0: + return None + return round(compute_audio_rtf(float(generation_ms) / 1000.0, float(duration_ms) / 1000.0), 6) + + def _video_frames_from_file( path: Path, *, @@ -156,7 +177,7 @@ def _resolve_video_frames(args: DemoArgs) -> tuple[list[str], list[str | None]]: @dataclass class DemoState: events: list[dict[str, object]] = field(default_factory=list) - timing_events: RealtimeEventCollector = field(default_factory=RealtimeEventCollector) + timing_events: EventCollector = field(default_factory=EventCollector) audio_deltas: list[bytes] = field(default_factory=list) response_audio_deltas: dict[str, list[bytes]] = field(default_factory=dict) response_ids: list[str] = field(default_factory=list) @@ -273,6 +294,11 @@ class DemoState: after_s=created_at_s, input_committed_at_s=input_committed_at_s, response_id=response_id, + measurement_origin={ + "ttft": "input_audio_buffer.commit client send to first non-empty text delta", + "ttfp": "input_audio_buffer.commit client send to first audio packet", + "rtf": "commit-to-last-audio receive time divided by emitted audio duration", + }, ) timing["measurement_origin"] = { "response": "response.created client receive", @@ -292,14 +318,14 @@ class DemoState: metrics = timing.get("request_metrics") if not isinstance(metrics, dict): continue - requests.append( - { - "session_id": session_id, - "request_index": request_index, - "response_id": response_id, - **metrics, - } - ) + request_metrics = { + "session_id": session_id, + "request_index": request_index, + "response_id": response_id, + **metrics, + "rtf": _audio_rtf_from_raw_metric(metrics), + } + requests.append(request_metrics) return requests def session_metric_summary( @@ -602,7 +628,7 @@ def _session_update_event(args: DemoArgs) -> dict[str, object]: "playback_commit_policy": "ack_only", "extra_body": { "auto_response": True, - "minicpmo45_native_duplex": True, + "native_duplex": True, "force_listen_count": 0, }, } diff --git a/tests/e2e/online_serving/personaplex_realtime_duplex.py b/tests/e2e/online_serving/personaplex_realtime_duplex.py index 62ad9e972..26ee22037 100644 --- a/tests/e2e/online_serving/personaplex_realtime_duplex.py +++ b/tests/e2e/online_serving/personaplex_realtime_duplex.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Strict PersonaPlex validation for the unified OpenAI Realtime duplex path.""" from __future__ import annotations @@ -18,17 +18,72 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import numpy as np -from vllm_omni.experimental.fullduplex.client import ( - RealtimeDuplexClient, - wait_for, +try: + import websockets + from websockets.exceptions import ConnectionClosed +except ImportError as exc: # pragma: no cover - driver dependency. + raise SystemExit("Install websockets first: pip install websockets") from exc + +from vllm_omni.clients.duplex import ( + EventCollector, write_pcm16_wav, ) +from vllm_omni.clients.duplex import ( + wait_for_condition as wait_for, +) SAMPLE_RATE_HZ = 24_000 FRAME_SAMPLES = 1_920 FRAME_PERIOD_S = FRAME_SAMPLES / SAMPLE_RATE_HZ +class RawRealtimeProbe: + """Raw-event Realtime probe: one websocket plus an :class:`EventCollector`. + + This driver speaks the wire protocol directly (it validates handshake + errors such as admission limits, which the public + :class:`vllm_omni.clients.duplex.DuplexClient` treats as failures), so it + keeps a transport-only client and reads everything from ``.events``. + """ + + def __init__(self, url: str, *, max_size: int = 64 * 1024 * 1024) -> None: + self.url = url + self.max_size = max_size + self.events = EventCollector() + self._ws = None + self._reader_task: asyncio.Task[None] | None = None + + async def __aenter__(self) -> RawRealtimeProbe: + self._ws = await websockets.connect(self.url, max_size=self.max_size) + self._reader_task = asyncio.create_task(self._read_events()) + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + if self._ws is not None: + await self._ws.close() + if self._reader_task is not None: + self._reader_task.cancel() + try: + await self._reader_task + except asyncio.CancelledError: + pass + + async def _read_events(self) -> None: + try: + while True: + raw = await self._ws.recv() + if not isinstance(raw, str): + continue + event = json.loads(raw) + if isinstance(event, dict): + self.events.add(event) + except ConnectionClosed: + return + + async def send(self, event: dict[str, object]) -> None: + await self._ws.send(json.dumps(event)) + + def _input_identity( path: Path, *, @@ -83,7 +138,7 @@ def _read_wav_as_float32(path: Path) -> np.ndarray: return pcm -def _events(client: RealtimeDuplexClient, event_type: str) -> list[dict[str, object]]: +def _events(client: RawRealtimeProbe, event_type: str) -> list[dict[str, object]]: return [event for event in client.events.events if event.get("type") == event_type] @@ -93,8 +148,8 @@ async def _open_session( session_id: str, persona: str, expect_error: bool = False, -) -> tuple[RealtimeDuplexClient, dict[str, object]]: - client = RealtimeDuplexClient(_realtime_url(args.url, args.model, session_id)) +) -> tuple[RawRealtimeProbe, dict[str, object]]: + client = RawRealtimeProbe(_realtime_url(args.url, args.model, session_id)) await client.__aenter__() await client.send( { @@ -121,7 +176,7 @@ async def _open_session( return client, _events(client, event_type)[-1] -async def _close_session(client: RealtimeDuplexClient, *, timeout_s: float) -> None: +async def _close_session(client: RawRealtimeProbe, *, timeout_s: float) -> None: await client.send({"type": "session.close"}) await wait_for( lambda: client.events.count("session.closed") > 0, @@ -132,7 +187,7 @@ async def _close_session(client: RealtimeDuplexClient, *, timeout_s: float) -> N async def _stream_frames( - client: RealtimeDuplexClient, + client: RawRealtimeProbe, pcm: np.ndarray, *, max_frames: int | None = None, @@ -186,7 +241,7 @@ def _audio_frame_stats( } -def _response_ids(client: RealtimeDuplexClient) -> set[str]: +def _response_ids(client: RawRealtimeProbe) -> set[str]: return { str(event["response_id"]) for event in _events(client, "response.audio.delta") @@ -195,7 +250,7 @@ def _response_ids(client: RealtimeDuplexClient) -> set[str]: def _session_result( - client: RealtimeDuplexClient, + client: RawRealtimeProbe, *, input_frames: int, args: argparse.Namespace, @@ -268,7 +323,7 @@ def _capabilities(created: dict[str, object]) -> dict[str, object]: return capabilities -def _save(output_dir: Path, name: str, client: RealtimeDuplexClient, raw: bytes) -> None: +def _save(output_dir: Path, name: str, client: RawRealtimeProbe, raw: bytes) -> None: (output_dir / f"{name}-events.jsonl").write_text( "".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events), encoding="utf-8", diff --git a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py index 910a5a180..32e6e01da 100644 --- a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py +++ b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py @@ -228,7 +228,7 @@ async def _open_admission_session( "session_id": session_id, "model": args.model, "modalities": ["audio", "text"], - "extra_body": {"minicpmo45_native_duplex": True}, + "extra_body": {"native_duplex": True}, **({"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) else {}), }, } @@ -271,7 +271,7 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str, "session_id": overflow_id, "model": args.model, "modalities": ["audio", "text"], - "extra_body": {"minicpmo45_native_duplex": True}, + "extra_body": {"native_duplex": True}, **( {"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) @@ -337,7 +337,7 @@ async def _resume_probe( "session_id": session_id, "model": args.model, "modalities": ["audio", "text"], - "extra_body": {"minicpmo45_native_duplex": True}, + "extra_body": {"native_duplex": True}, **( {"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) @@ -442,7 +442,7 @@ async def _takeover_probe( "session_id": session_id, "model": args.model, "modalities": ["audio", "text"], - "extra_body": {"minicpmo45_native_duplex": True}, + "extra_body": {"native_duplex": True}, **( {"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) diff --git a/tests/e2e/online_serving/test_duplex_client_live.py b/tests/e2e/online_serving/test_duplex_client_live.py new file mode 100644 index 000000000..d90a8a683 --- /dev/null +++ b/tests/e2e/online_serving/test_duplex_client_live.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Live E2E coverage for the public duplex client (``vllm_omni.clients.duplex``). + +Drives a real MiniCPM-o 4.5 duplex session end to end through +:class:`DuplexClient`: session handshake with reference audio, paced PCM +streaming, a speak response consumed via :class:`ResponseHandle` with +incremental playback acks, a forced transport drop recovered by automatic +``session.resume``, and a clean close. +""" + +from __future__ import annotations + +import asyncio +import uuid +from pathlib import Path + +import pytest + +from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import ( + CORE_SERVER_PARAMS, + realtime_url, + resolve_ref_audio, + validated_input_wav, +) +from tests.helpers.mark import hardware_test +from vllm_omni.clients.duplex import ( + DuplexClient, + EventCollector, + ReconnectPolicy, + audio_data_url, + read_pcm16_wav, +) +from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config + +pytestmark = pytest.mark.omni + +_SESSION_TIMEOUT_S = 900.0 +_MAX_LISTEN_ROUNDS = 30 + + +async def _run_live_session(*, url: str, model: str, ref_audio: Path, input_wav: Path) -> dict[str, object]: + # temperature 0.0 keeps the response-required reply short and + # deterministic (same as the validated demo driver). + config = create_duplex_session_config(ref_audio=audio_data_url(ref_audio), temperature=0.0) + client = DuplexClient( + url, + model=model, + config=config, + session_id=f"duplex-client-live-{uuid.uuid4().hex}", + reconnect=ReconnectPolicy(max_attempts=5, backoff_s=(0.25, 1.0)), + # Short interval so the periodic heartbeat path runs during the + # session — including across the forced transport drop below. + heartbeat_interval_s=5.0, + ) + collector = EventCollector() + pcm = read_pcm16_wav(input_wav) + summary: dict[str, object] = {} + async with client: + collector_task = asyncio.create_task(collector.consume(client)) + summary["session_id"] = client.session_id + summary["resume_token_issued"] = client.resume_token is not None + + # Phase 1: stream the validated utterance and consume one speak response. + await client.stream_pcm(pcm, chunk_ms=200) + await client.commit() + listens = 0 + async for response in client.responses(): + if response.decision == "listen": + listens += 1 + if listens > _MAX_LISTEN_ROUNDS: + raise AssertionError(f"no speak response after {listens} listen decisions") + continue + chunk_count = 0 + audio_byte_count = 0 + async for chunk in response.audio(): + chunk_count += 1 + audio_byte_count += len(chunk) + await client.ack_playback(response.played_ms) + summary.update( + { + "decision": response.decision, + "listen_rounds": listens, + "audio_chunks": chunk_count, + "audio_bytes": audio_byte_count, + "played_ms": response.played_ms, + "transcript": response.transcript, + } + ) + break + + # Phase 2: force-drop the transport underneath the client and require + # the automatic session.resume path to bring the session back. + resumed_waiter = asyncio.create_task(client.wait_for("session.resumed", timeout_s=90.0)) + await asyncio.sleep(0.1) # let the waiter subscribe before the drop + assert client._ws is not None + await client._ws.close() + resumed = await resumed_waiter + summary["resumed_session_id"] = resumed.session_id or client.session_id + + # The resumed transport must be live: a heartbeat gets acknowledged. + ack_waiter = asyncio.create_task(client.wait_for("session.heartbeat_ack", timeout_s=30.0)) + await asyncio.sleep(0.1) + await client.send({"type": "session.heartbeat"}) + await ack_waiter + summary["post_resume_heartbeat_ok"] = True + + await client.close() + await asyncio.wait_for(collector_task, timeout=10.0) + + summary["error_events"] = collector.errors() + summary["closed_cleanly"] = collector.count("session.closed") > 0 + summary["timing"] = collector.timing_summary(after_s=0.0) + return summary + + +@pytest.mark.core_model +@hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.parametrize("omni_server", CORE_SERVER_PARAMS, indirect=True) +def test_duplex_client_live_session(omni_server) -> None: + summary = asyncio.run( + asyncio.wait_for( + _run_live_session( + url=realtime_url(omni_server), + model=omni_server.model, + ref_audio=resolve_ref_audio(), + input_wav=validated_input_wav(), + ), + timeout=_SESSION_TIMEOUT_S, + ) + ) + assert summary["decision"] == "speak", summary + assert summary["audio_chunks"] > 0, summary + assert summary["played_ms"] > 0, summary + assert summary["transcript"], summary + assert summary["resume_token_issued"] is True, summary + assert summary["post_resume_heartbeat_ok"] is True, summary + assert summary["closed_cleanly"] is True, summary + assert summary["error_events"] == [], summary diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py b/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py index bbdf85c09..a8ed72511 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py @@ -30,7 +30,7 @@ from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import run_multi_session, ) from tests.helpers.mark import hardware_test -from vllm_omni.experimental.fullduplex.client import build_realtime_url +from vllm_omni.clients.duplex import build_realtime_url from vllm_omni.experimental.fullduplex.video_stacking import concat_frames_b64 pytestmark = pytest.mark.omni @@ -88,7 +88,13 @@ async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: f async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[dict[str, object]]: session_id = f"duplex-ci-protocol-{uuid.uuid4().hex}" - websocket_url = build_realtime_url(url, model, autostart=False, session_id=session_id) + websocket_url = build_realtime_url( + url, + model, + autostart=False, + session_id=session_id, + extra_query={"native_duplex": "1"}, + ) async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws: await ws.send( json.dumps( @@ -99,7 +105,7 @@ async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[ "model": model, "modalities": ["audio", "text"], "ref_audio": _ref_audio_data_url(str(ref_audio)), - "extra_body": {"minicpmo45_native_duplex": True}, + "extra_body": {"native_duplex": True}, }, } ) diff --git a/tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py b/tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py index 9569078d2..22c8e6b34 100644 --- a/tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py +++ b/tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py @@ -1001,7 +1001,7 @@ def test_realtime_duplex_demo_explicitly_enables_native_runtime_before_connect() ) query = parse_qs(urlsplit(url).query) - assert query["minicpmo45_native_duplex"] == ["1"] + assert query["native_duplex"] == ["1"] def test_realtime_duplex_demo_explicit_session_id_reaches_autostart_query(): @@ -1032,7 +1032,7 @@ def test_realtime_duplex_demo_session_update_uses_explicit_session_id(): assert event["type"] == "session.update" assert "session_id" not in event assert event["session"]["session_id"] == "reopen-e2e" - assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True + assert event["session"]["extra_body"]["native_duplex"] is True def test_realtime_duplex_demo_response_required_uses_deterministic_sampling(): diff --git a/tests/e2e/features/fullduplex/minicpmo45/__init__.py b/tests/engine/duplex/__init__.py similarity index 100% rename from tests/e2e/features/fullduplex/minicpmo45/__init__.py rename to tests/engine/duplex/__init__.py diff --git a/tests/e2e/features/fullduplex/engine/test_duplex_control_client.py b/tests/engine/duplex/test_duplex_control_client.py similarity index 92% rename from tests/e2e/features/fullduplex/engine/test_duplex_control_client.py rename to tests/engine/duplex/test_duplex_control_client.py index c1fa9a592..7974b14c9 100644 --- a/tests/e2e/features/fullduplex/engine/test_duplex_control_client.py +++ b/tests/engine/duplex/test_duplex_control_client.py @@ -1,16 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import pytest -from vllm_omni.experimental.fullduplex.engine.duplex_control_client import ( +from vllm_omni.engine.duplex.control_client import ( DuplexControlClient, DuplexControlRequestError, ) -from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity -from vllm_omni.experimental.fullduplex.engine.messages import ( +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity +from vllm_omni.engine.duplex.messages import ( DuplexControlError, DuplexControlResultMessage, DuplexFence, diff --git a/tests/e2e/features/fullduplex/engine/test_duplex_control_plane.py b/tests/engine/duplex/test_duplex_control_plane.py similarity index 99% rename from tests/e2e/features/fullduplex/engine/test_duplex_control_plane.py rename to tests/engine/duplex/test_duplex_control_plane.py index 46df33ffa..14da4d21c 100644 --- a/tests/e2e/features/fullduplex/engine/test_duplex_control_plane.py +++ b/tests/engine/duplex/test_duplex_control_plane.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -10,7 +10,7 @@ from dataclasses import FrozenInstanceError import msgspec import pytest -from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import ( +from vllm_omni.engine.duplex.control_plane import ( DuplexControlPlane, DuplexOutputContext, DuplexRequestIdentity, @@ -18,13 +18,8 @@ from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import ( DuplexStageSubmission, DuplexStageSubmissionResult, ) -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( - DuplexAppendPlan, - DuplexInputMode, - DuplexRuntimeCapabilities, -) -from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity, DuplexLeaseConfig -from vllm_omni.experimental.fullduplex.engine.messages import ( +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity, DuplexLeaseConfig +from vllm_omni.engine.duplex.messages import ( AppendDuplexInputMessage, CloseDuplexSessionMessage, DuplexFence, @@ -34,6 +29,11 @@ from vllm_omni.experimental.fullduplex.engine.messages import ( SignalDuplexTurnMessage, TouchDuplexSessionMessage, ) +from vllm_omni.engine.duplex.runtime import ( + DuplexAppendPlan, + DuplexInputMode, + DuplexRuntimeCapabilities, +) pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py b/tests/engine/duplex/test_duplex_deploy_config.py similarity index 97% rename from tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py rename to tests/engine/duplex/test_duplex_deploy_config.py index 65d4f1ca9..800070efb 100644 --- a/tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py +++ b/tests/engine/duplex/test_duplex_deploy_config.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Focused tests for typed deployment-only runtime configuration.""" from __future__ import annotations diff --git a/tests/e2e/features/fullduplex/engine/test_duplex_intermediate.py b/tests/engine/duplex/test_duplex_intermediate.py similarity index 69% rename from tests/e2e/features/fullduplex/engine/test_duplex_intermediate.py rename to tests/engine/duplex/test_duplex_intermediate.py index 37602e62e..d542f3cab 100644 --- a/tests/e2e/features/fullduplex/engine/test_duplex_intermediate.py +++ b/tests/engine/duplex/test_duplex_intermediate.py @@ -1,6 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import pytest -from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key +from vllm_omni.engine.duplex.intermediate import get_stream_request_key pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/e2e/features/fullduplex/engine/test_duplex_lease.py b/tests/engine/duplex/test_duplex_lease.py similarity index 95% rename from tests/e2e/features/fullduplex/engine/test_duplex_lease.py rename to tests/engine/duplex/test_duplex_lease.py index 80e7a73eb..62fc8c70c 100644 --- a/tests/e2e/features/fullduplex/engine/test_duplex_lease.py +++ b/tests/engine/duplex/test_duplex_lease.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -7,13 +7,13 @@ from dataclasses import FrozenInstanceError import pytest -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import DuplexInputMode -from vllm_omni.experimental.fullduplex.engine.duplex_session import DuplexSessionRuntimeManager -from vllm_omni.experimental.fullduplex.engine.lease import ( +from vllm_omni.engine.duplex.lease import ( DuplexLeaseActivity, DuplexLeaseConfig, ) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import DuplexInputMode +from vllm_omni.engine.duplex.session import DuplexSessionRuntimeManager pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/e2e/features/fullduplex/engine/test_duplex_runtime.py b/tests/engine/duplex/test_duplex_runtime.py similarity index 97% rename from tests/e2e/features/fullduplex/engine/test_duplex_runtime.py rename to tests/engine/duplex/test_duplex_runtime.py index 7cce93161..9cbf4d6e1 100644 --- a/tests/e2e/features/fullduplex/engine/test_duplex_runtime.py +++ b/tests/engine/duplex/test_duplex_runtime.py @@ -1,24 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from dataclasses import FrozenInstanceError from types import SimpleNamespace import pytest from vllm.sampling_params import SamplingParams -from vllm_omni.experimental.fullduplex.engine import duplex_runtime -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex import runtime as duplex_runtime +from vllm_omni.engine.duplex.contracts import ( duplex_data_plane_request_info, duplex_resource_request_belongs_to_session, duplex_resource_request_id, ) -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import ( DuplexInputMode, DuplexOutputAction, DuplexOutputDecision, DuplexRuntimeCapabilities, DuplexSessionRuntimeManager, ) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.minicpmo45.runtime import ( +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import ( MiniCPMO45DuplexRuntimeExtension, build_duplex_data_plane_prompt, duplex_scheduler_token_budget, diff --git a/tests/engine/test_async_omni_engine_outputs.py b/tests/engine/test_async_omni_engine_outputs.py index a1b7a536e..6b0d6a188 100644 --- a/tests/engine/test_async_omni_engine_outputs.py +++ b/tests/engine/test_async_omni_engine_outputs.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """Tests for AsyncOmniEngine.try_get_output and try_get_output_async. Focuses on the critical behavior: when the orchestrator thread dies, @@ -15,22 +18,22 @@ from pytest_mock import MockerFixture from vllm_omni.engine.async_engine_utils import weak_shutdown_async_omni_engine from vllm_omni.engine.async_omni_engine import AsyncOmniEngine +from vllm_omni.engine.duplex.control_client import ( + DuplexControlClient, + DuplexControlRequestError, +) +from vllm_omni.engine.duplex.messages import ( + AppendDuplexInputMessage, + DuplexControlResultMessage, + DuplexFence, + SignalDuplexTurnMessage, +) from vllm_omni.engine.messages import ( CollectiveRPCResultMessage, ErrorMessage, OutputMessage, ) from vllm_omni.engine.rpc_result_router import CorrelatedRpcClient -from vllm_omni.experimental.fullduplex.engine.duplex_control_client import ( - DuplexControlClient, - DuplexControlRequestError, -) -from vllm_omni.experimental.fullduplex.engine.messages import ( - AppendDuplexInputMessage, - DuplexControlResultMessage, - DuplexFence, - SignalDuplexTurnMessage, -) from vllm_omni.outputs import OmniRequestOutput pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/engine/test_duplex_import_boundary.py b/tests/engine/test_duplex_import_boundary.py index d04ed2e70..f30266d28 100644 --- a/tests/engine/test_duplex_import_boundary.py +++ b/tests/engine/test_duplex_import_boundary.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -29,7 +29,9 @@ def _assert_isolated_import_succeeds(script: str) -> None: env=env, capture_output=True, text=True, - timeout=60, + # The CLI probe imports the whole vllm + benchmark graph; on a loaded + # shared host that alone can take over a minute. + timeout=180, check=False, ) assert result.returncode == 0, result.stderr @@ -43,7 +45,11 @@ def test_isolated_import_ignores_conflicting_gpu_visibility(monkeypatch: pytest. _assert_isolated_import_succeeds("import vllm_omni.outputs") -def test_stable_engine_imports_do_not_load_experimental_duplex() -> None: +def test_stable_engine_imports_load_duplex_kernel_eagerly() -> None: + # The duplex kernel is imported eagerly by the stable engine modules. + # Model-specific duplex adapters must still load only via the + # dotted-string plugin paths, and the client-side vllm_omni.clients + # package must never enter the stable runtime import graph. _assert_isolated_import_succeeds(""" import sys @@ -51,14 +57,84 @@ import vllm_omni.engine.async_omni_engine import vllm_omni.engine.orchestrator import vllm_omni.entrypoints.async_omni +expected_eager = ( + "vllm_omni.engine.duplex.contracts", + "vllm_omni.engine.duplex.control_plane", + "vllm_omni.entrypoints.duplex_request_client", + "vllm_omni.outputs.duplex", +) +missing = sorted(name for name in expected_eager if name not in sys.modules) +if missing: + raise SystemExit("duplex kernel modules not imported eagerly: " + ", ".join(missing)) + +forbidden_prefixes = ( + "vllm_omni.clients", + "vllm_omni.model_executor.models.minicpmo_4_5.duplex", + "vllm_omni.model_executor.models.nemotron_voicechat.duplex", + "vllm_omni.model_executor.models.personaplex.duplex", +) loaded = sorted( name for name in sys.modules - if name == "vllm_omni.experimental.fullduplex" - or name.startswith("vllm_omni.experimental.fullduplex.") + if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes) ) if loaded: - raise SystemExit("stable imports loaded experimental duplex: " + ", ".join(loaded)) + raise SystemExit("stable imports loaded plugin-only duplex modules: " + ", ".join(loaded)) +""") + + +def test_cli_import_graph_stays_clear_of_the_client_package() -> None: + # `vllm-omni serve` imports the CLI package, whose __init__ eagerly pulls + # in the benchmark patch module and every benchmark subcommand. None of + # that may drag the client-side vllm_omni.clients package into the server + # process; the benchmark runtime imports it lazily at execution time. + _assert_isolated_import_succeeds(""" +import sys + +import vllm_omni.entrypoints.cli.main + +loaded = sorted( + name + for name in sys.modules + if name == "vllm_omni.clients" or name.startswith("vllm_omni.clients.") +) +if loaded: + raise SystemExit("CLI import graph loaded the client package: " + ", ".join(loaded)) +""") + + +def test_client_package_stays_clear_of_the_server_import_graph() -> None: + # vllm_omni.clients.duplex advertises a lightweight dependency set + # (pybase64 + websockets). Importing it and exercising the code paths + # that previously reached into the server tree (timing_summary once + # imported vllm_omni.metrics, dragging in prometheus_client) must not + # load any vllm_omni module outside the client package. + _assert_isolated_import_succeeds(""" +import base64 +import sys + +import vllm_omni # the package __init__ loads its own baseline graph + +baseline = set(sys.modules) + +from vllm_omni.clients.duplex import EventCollector + +collector = EventCollector() +chunk = base64.b64encode(b"\\x00\\x00" * 2400).decode("ascii") +collector.add({"type": "response.created", "response_id": "r1"}, received_at_s=1.0) +collector.add({"type": "response.audio.delta", "response_id": "r1", "delta": chunk}, received_at_s=1.1) +collector.add({"type": "response.audio.delta", "response_id": "r1", "delta": chunk}, received_at_s=1.2) +summary = collector.timing_summary(after_s=0.0, response_id="r1") +if "audio_output" not in summary or "request_metrics" not in summary: + raise SystemExit("timing_summary produced no audio_output/request_metrics sections") + +added = sorted( + name + for name in set(sys.modules) - baseline + if name.startswith("vllm_omni.") and not name.startswith("vllm_omni.clients") +) +if added: + raise SystemExit("the client package pulled in server modules: " + ", ".join(added)) """) @@ -95,26 +171,43 @@ if "duplex_output_decision" in fields: """) -def test_stable_model_executor_does_not_expose_duplex_helper_module() -> None: +def test_duplex_sampling_helper_lives_in_model_executor() -> None: _assert_isolated_import_succeeds(""" import importlib.util -if importlib.util.find_spec("vllm_omni.model_executor.duplex") is not None: - raise SystemExit("stable model_executor still exposes duplex helper module") +if importlib.util.find_spec("vllm_omni.model_executor.duplex_sampling") is None: + raise SystemExit("model_executor duplex_sampling helper module is missing") """) + assert not (REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "model_executor.py").exists() def test_runtime_package_does_not_bundle_the_browser_demo() -> None: assert not (REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "web").exists() -def test_experimental_engine_uses_canonical_contract_module_names() -> None: - engine_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "engine" +def test_engine_duplex_uses_canonical_contract_module_names() -> None: + engine_dir = REPO_ROOT / "vllm_omni" / "engine" / "duplex" core_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "core" - assert (engine_dir / "contracts.py").is_file() - assert (engine_dir / "lease.py").is_file() - assert (engine_dir / "messages.py").is_file() + for name in ( + "contracts.py", + "lease.py", + "messages.py", + "session.py", + "control_plane.py", + "control_client.py", + "runtime.py", + "intermediate.py", + ): + assert (engine_dir / name).is_file() + # the duplex_ prefix is dropped inside the duplex package + assert not (engine_dir / "duplex_session.py").exists() assert not (engine_dir / "duplex_lease.py").exists() assert not (engine_dir / "duplex_types.py").exists() + # the experimental package retains only the joyvl framework; stale + # __pycache__ leftovers may exist, so assert on source files, not the dir + old_engine_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "engine" + assert not list(old_engine_dir.glob("*.py")) + old_personaplex_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "personaplex" + assert not list(old_personaplex_dir.glob("**/*.py")) assert not (core_dir / "identity.py").exists() diff --git a/tests/engine/test_orchestrator.py b/tests/engine/test_orchestrator.py index 9dcf066cd..99b81aeb9 100644 --- a/tests/engine/test_orchestrator.py +++ b/tests/engine/test_orchestrator.py @@ -21,6 +21,20 @@ from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs, FinishReason from vllm.v1.engine.exceptions import EngineDeadError from vllm.v1.metrics.stats import IterationStats +from vllm_omni.engine.duplex.control_plane import DuplexControlPlane +from vllm_omni.engine.duplex.messages import ( + AppendDuplexInputMessage, + CloseDuplexSessionMessage, + DuplexFence, + OpenDuplexSessionMessage, + SignalDuplexTurnMessage, +) +from vllm_omni.engine.duplex.runtime import ( + DuplexInputMode, + DuplexRuntimeCapabilities, + DuplexSessionRuntimeState, + duplex_resource_request_id, +) from vllm_omni.engine.messages import ( AbortRequestMessage, AbortResultMessage, @@ -39,22 +53,8 @@ from vllm_omni.engine.orchestrator import ( _build_terminal_empty_output, ) from vllm_omni.engine.stage_pool import StagePool -from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import DuplexControlPlane -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( - DuplexInputMode, - DuplexRuntimeCapabilities, - DuplexSessionRuntimeState, - duplex_resource_request_id, -) -from vllm_omni.experimental.fullduplex.engine.messages import ( - AppendDuplexInputMessage, - CloseDuplexSessionMessage, - DuplexFence, - OpenDuplexSessionMessage, - SignalDuplexTurnMessage, -) -from vllm_omni.experimental.fullduplex.minicpmo45.runtime import MiniCPMO45DuplexRuntimeExtension from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import MiniCPMO45DuplexRuntimeExtension from vllm_omni.outputs import OmniRequestOutput pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/engine/test_orchestrator_segment_stage_keying.py b/tests/engine/test_orchestrator_segment_stage_keying.py index 02bb9a338..5d3952e4b 100644 --- a/tests/engine/test_orchestrator_segment_stage_keying.py +++ b/tests/engine/test_orchestrator_segment_stage_keying.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Streaming segment-boundary state must be keyed by stage. ``Orchestrator._orchestration_loop`` polls every stage into the same @@ -15,13 +15,13 @@ from dataclasses import dataclass, field import pytest from vllm.sampling_params import SamplingParams +from vllm_omni.engine.duplex.contracts import DuplexRequestIdentity +from vllm_omni.engine.duplex.messages import DuplexFence from vllm_omni.engine.orchestrator import ( Orchestrator, OrchestratorRequestState, StreamingSegmentState, ) -from vllm_omni.experimental.fullduplex.engine.contracts import DuplexRequestIdentity -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/engine/test_orchestrator_stage_input_bridge.py b/tests/engine/test_orchestrator_stage_input_bridge.py index 52cee4657..68836e2a4 100644 --- a/tests/engine/test_orchestrator_stage_input_bridge.py +++ b/tests/engine/test_orchestrator_stage_input_bridge.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -14,6 +14,11 @@ import pytest from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import SamplingParams +from vllm_omni.engine.duplex.contracts import ( + DuplexStageRequestContext, + DuplexStageSubmission, +) +from vllm_omni.engine.duplex.messages import DuplexFence from vllm_omni.engine.orchestrator import ( Orchestrator, OrchestratorRequestState, @@ -21,11 +26,6 @@ from vllm_omni.engine.orchestrator import ( _OrchestratorDuplexStagePort, ) from vllm_omni.engine.stage_pool import StagePool -from vllm_omni.experimental.fullduplex.engine.contracts import ( - DuplexStageRequestContext, - DuplexStageSubmission, -) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/e2e/features/fullduplex/openai/test_duplex_audio.py b/tests/entrypoints/duplex/test_duplex_audio.py similarity index 91% rename from tests/e2e/features/fullduplex/openai/test_duplex_audio.py rename to tests/entrypoints/duplex/test_duplex_audio.py index 535856976..d0be02072 100644 --- a/tests/e2e/features/fullduplex/openai/test_duplex_audio.py +++ b/tests/entrypoints/duplex/test_duplex_audio.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import base64 import numpy as np import pytest -from vllm_omni.experimental.fullduplex.openai.audio import convert_input_audio_with_rate +from vllm_omni.entrypoints.duplex.audio import convert_input_audio_with_rate pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/e2e/features/fullduplex/openai/test_runtime_adapter_boundary.py b/tests/entrypoints/duplex/test_runtime_adapter_boundary.py similarity index 79% rename from tests/e2e/features/fullduplex/openai/test_runtime_adapter_boundary.py rename to tests/entrypoints/duplex/test_runtime_adapter_boundary.py index d7cecc13d..dc7fffccd 100644 --- a/tests/e2e/features/fullduplex/openai/test_runtime_adapter_boundary.py +++ b/tests/entrypoints/duplex/test_runtime_adapter_boundary.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -10,7 +10,7 @@ from types import SimpleNamespace import pytest -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( +from vllm_omni.entrypoints.duplex.runtime_adapter import ( validate_serving_runtime_adapter, ) @@ -20,26 +20,33 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @pytest.mark.parametrize( "module_name", [ - "vllm_omni.experimental.fullduplex.openai.session_runner", - "vllm_omni.experimental.fullduplex.openai.runtime_bridge", - "vllm_omni.experimental.fullduplex.openai.serving", + "vllm_omni.entrypoints.duplex.session_runner", + "vllm_omni.entrypoints.duplex.runtime_bridge", + "vllm_omni.entrypoints.duplex.serving", ], ) -def test_generic_openai_runtime_import_does_not_load_minicpmo45(module_name: str) -> None: +def test_generic_openai_runtime_import_does_not_load_model_adapters(module_name: str) -> None: repo_root = Path(__file__).resolve().parents[3] script = f""" import sys import {module_name} +model_duplex_prefixes = ( + "vllm_omni.model_executor.models.minicpmo_4_5.duplex", + "vllm_omni.model_executor.models.nemotron_voicechat.duplex", + "vllm_omni.model_executor.models.personaplex.duplex", +) loaded = sorted( name for name in sys.modules - if name == "vllm_omni.experimental.fullduplex.minicpmo45" - or name.startswith("vllm_omni.experimental.fullduplex.minicpmo45.") + if any( + name == prefix or name.startswith(prefix + ".") + for prefix in model_duplex_prefixes + ) ) if loaded: - raise SystemExit("generic OpenAI runtime loaded MiniCPM-o: " + ", ".join(loaded)) + raise SystemExit("generic OpenAI runtime loaded model duplex modules: " + ", ".join(loaded)) """ result = subprocess.run( @@ -59,7 +66,7 @@ def test_generic_handler_requires_explicit_serving_runtime_adapter() -> None: script = """ from types import SimpleNamespace -from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler +from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler chat_service = SimpleNamespace(engine_client=SimpleNamespace()) try: diff --git a/tests/e2e/features/fullduplex/openai/test_runtime_control_redaction.py b/tests/entrypoints/duplex/test_runtime_control_redaction.py similarity index 93% rename from tests/e2e/features/fullduplex/openai/test_runtime_control_redaction.py rename to tests/entrypoints/duplex/test_runtime_control_redaction.py index 16bea686d..cfc5b68c5 100644 --- a/tests/e2e/features/fullduplex/openai/test_runtime_control_redaction.py +++ b/tests/entrypoints/duplex/test_runtime_control_redaction.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Regression tests for duplex ``runtime_control`` payload redaction. @@ -19,8 +19,8 @@ from enum import Enum import pytest -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.openai.runtime_bridge import NativeRuntimeBridgeMixin +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.entrypoints.duplex.runtime_bridge import NativeRuntimeBridgeMixin pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/e2e/features/fullduplex/openai/test_websocket_actor.py b/tests/entrypoints/duplex/test_websocket_actor.py similarity index 97% rename from tests/e2e/features/fullduplex/openai/test_websocket_actor.py rename to tests/entrypoints/duplex/test_websocket_actor.py index 37924af52..9bdb73667 100644 --- a/tests/e2e/features/fullduplex/openai/test_websocket_actor.py +++ b/tests/entrypoints/duplex/test_websocket_actor.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import asyncio import pytest -from vllm_omni.experimental.fullduplex.openai.websocket import ( +from vllm_omni.entrypoints.duplex.websocket import ( DuplexWebSocketActor, normalize_duplex_input_event, ) diff --git a/tests/entrypoints/openai/test_duplex_protocol.py b/tests/entrypoints/openai/test_duplex_protocol.py index 9fd61095b..e5f3becbc 100644 --- a/tests/entrypoints/openai/test_duplex_protocol.py +++ b/tests/entrypoints/openai/test_duplex_protocol.py @@ -1,10 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import asyncio import json import pytest -from vllm_omni.experimental.fullduplex.openai.protocol import ( - DuplexCapabilities, +from vllm_omni.entrypoints.duplex.protocol import ( DuplexOverlapPolicy, DuplexSession, DuplexSessionConfig, @@ -14,10 +16,13 @@ from vllm_omni.experimental.fullduplex.openai.protocol import ( DuplexTurnState, ResponseCreateOptions, ) -from vllm_omni.experimental.fullduplex.openai.realtime_session import ( +from vllm_omni.entrypoints.duplex.realtime_session import ( NativeRealtimeSessionProtocol, ) -from vllm_omni.experimental.fullduplex.openai.realtime_state import RealtimeStateOwner +from vllm_omni.entrypoints.duplex.realtime_state import RealtimeStateOwner +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import ( + minicpmo45_native_capabilities, +) pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -116,12 +121,28 @@ def test_realtime_model_name_does_not_implicitly_enable_native_duplex(): event = protocol._session_create_from_realtime({"model": "openbmb/MiniCPM-o-4_5"}) + assert "native_duplex" not in event["session"]["extra_body"] assert "minicpmo45_native_duplex" not in event["session"]["extra_body"] def test_realtime_explicit_native_duplex_flag_is_preserved(): protocol = NativeRealtimeSessionProtocol({}) + event = protocol._session_create_from_realtime( + { + "model": "openbmb/MiniCPM-o-4_5", + "extra_body": {"native_duplex": True}, + } + ) + + assert event["session"]["extra_body"]["native_duplex"] is True + + +def test_realtime_legacy_native_duplex_alias_is_normalized(): + # The deprecated model-prefixed spelling still opts in, but everything + # downstream (and every session echo) sees only the canonical key. + protocol = NativeRealtimeSessionProtocol({}) + event = protocol._session_create_from_realtime( { "model": "openbmb/MiniCPM-o-4_5", @@ -129,10 +150,25 @@ def test_realtime_explicit_native_duplex_flag_is_preserved(): } ) - assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True + assert event["session"]["extra_body"]["native_duplex"] is True + assert "minicpmo45_native_duplex" not in event["session"]["extra_body"] def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostart(): + protocol = NativeRealtimeSessionProtocol( + { + "model": "openbmb/MiniCPM-o-4_5", + "native_duplex": "1", + } + ) + + event = json.loads(asyncio.run(protocol.receive_internal_event_text(None))) + + assert event["type"] == "session.create" + assert event["session"]["extra_body"]["native_duplex"] is True + + +def test_realtime_legacy_query_native_duplex_alias_seeds_canonical_key(): protocol = NativeRealtimeSessionProtocol( { "model": "openbmb/MiniCPM-o-4_5", @@ -143,7 +179,8 @@ def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostar event = json.loads(asyncio.run(protocol.receive_internal_event_text(None))) assert event["type"] == "session.create" - assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True + assert event["session"]["extra_body"]["native_duplex"] is True + assert "minicpmo45_native_duplex" not in event["session"]["extra_body"] @pytest.mark.asyncio @@ -151,7 +188,7 @@ async def test_realtime_resume_heartbeat_and_event_ack_translate_without_session protocol = NativeRealtimeSessionProtocol( { "model": "openbmb/MiniCPM-o-4_5", - "minicpmo45_native_duplex": "1", + "native_duplex": "1", } ) @@ -217,7 +254,7 @@ async def test_realtime_resume_query_suppresses_model_autostart_on_same_url(): protocol = NativeRealtimeSessionProtocol( { "model": "openbmb/MiniCPM-o-4_5", - "minicpmo45_native_duplex": "1", + "native_duplex": "1", "resume": "1", } ) @@ -293,6 +330,39 @@ def test_realtime_projects_resume_lifecycle_events_without_duplex_prefix(): } +def test_realtime_projects_listen_response_identity(): + protocol = NativeRealtimeSessionProtocol({}) + protocol.encode_outbound_event({"type": "session.created", "session": {"id": "sid-listen"}}) + + bound = protocol.encode_outbound_event( + { + "type": "response.listen", + "session_id": "sid-listen", + "epoch": 0, + "reason": "model_listen", + "model_listen": True, + "response_id": "resp_1", + } + )[0] + # A listen that terminates a precreated response keeps the response + # identity on the Realtime dialect; clients demultiplex on response_id. + assert bound["type"] == "response.listen" + assert bound["response_id"] == "resp_1" + assert bound["response"]["id"] == "resp_1" + assert bound["response"]["status"] == "listening" + + anonymous = protocol.encode_outbound_event( + { + "type": "response.listen", + "session_id": "sid-listen", + "epoch": 0, + "reason": "silence_or_noise", + } + )[0] + assert "response_id" not in anonymous + assert "id" not in anonymous["response"] + + def test_duplex_session_registry_advances_incarnation_when_id_is_reused(): registry = DuplexSessionRegistry() @@ -436,7 +506,7 @@ def test_duplex_capabilities_do_not_claim_core_kv_or_input_append(): def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease(): - caps = DuplexCapabilities.minicpmo45_native(max_sessions=2).as_dict() + caps = minicpmo45_native_capabilities(max_sessions=2).as_dict() assert caps["implementation_level"] == "model_native_duplex" assert caps["supports_input_append"] is True @@ -463,7 +533,7 @@ def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease(): def test_duplex_overlap_policy_auto_falls_back_to_listen_only(): - from vllm_omni.experimental.fullduplex.openai.protocol import ( + from vllm_omni.entrypoints.duplex.protocol import ( DuplexOverlapPolicy, DuplexSessionConfig, ) @@ -472,7 +542,7 @@ def test_duplex_overlap_policy_auto_falls_back_to_listen_only(): def test_minicpmo_native_capabilities_do_not_overclaim_single_session_deployment(): - caps = DuplexCapabilities.minicpmo45_native(max_sessions=1).as_dict() + caps = minicpmo45_native_capabilities(max_sessions=1).as_dict() assert caps["supports_multi_session"] is False assert caps["supports_multi_session_same_replica"] is False diff --git a/tests/entrypoints/openai/test_duplex_session_attachment.py b/tests/entrypoints/openai/test_duplex_session_attachment.py index 24a6f5365..b4ea9c328 100644 --- a/tests/entrypoints/openai/test_duplex_session_attachment.py +++ b/tests/entrypoints/openai/test_duplex_session_attachment.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -9,7 +9,7 @@ import json import pytest -from vllm_omni.experimental.fullduplex.openai.session_attachment import ( +from vllm_omni.entrypoints.duplex.session_attachment import ( DuplexEventJournal, DuplexJournalGapError, DuplexJournalOverflowError, @@ -43,7 +43,7 @@ def test_resume_credential_uses_256_bit_token_digest_and_redacted_repr(monkeypat credential = DuplexResumeCredential.from_token(token) calls = [] - import vllm_omni.experimental.fullduplex.openai.session_attachment as attachment_module + import vllm_omni.entrypoints.duplex.session_attachment as attachment_module original_compare = attachment_module.hmac.compare_digest diff --git a/tests/entrypoints/openai_api/test_duplex_capability.py b/tests/entrypoints/openai_api/test_duplex_capability.py index 153563cbb..21ee7e656 100644 --- a/tests/entrypoints/openai_api/test_duplex_capability.py +++ b/tests/entrypoints/openai_api/test_duplex_capability.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import ast @@ -7,13 +10,37 @@ import pytest pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +_REPO_ROOT = Path(__file__).parents[3] +_CLIENTS_PACKAGE = "vllm_omni.clients" -def test_api_server_does_not_import_experimental_duplex_at_module_load() -> None: - api_server = Path(__file__).parents[3] / "vllm_omni/entrypoints/openai/api_server.py" - module = ast.parse(api_server.read_text()) - top_level_imports = { - node.module for node in module.body if isinstance(node, ast.ImportFrom) and isinstance(node.module, str) - } +def _imports_clients_package(path: Path) -> bool: + """True if the module imports the client-side package at module level. - assert not any(module.startswith("vllm_omni.experimental.fullduplex") for module in top_level_imports) + ``vllm_omni.clients`` is client-side by construction: the serving stack + (imported eagerly by the API server) must never depend on it. + """ + module = ast.parse(path.read_text(encoding="utf-8")) + for node in module.body: + if isinstance(node, ast.Import): + if any( + alias.name == _CLIENTS_PACKAGE or alias.name.startswith(_CLIENTS_PACKAGE + ".") for alias in node.names + ): + return True + elif isinstance(node, ast.ImportFrom): + if node.module and (node.module == _CLIENTS_PACKAGE or node.module.startswith(_CLIENTS_PACKAGE + ".")): + return True + if node.module == "vllm_omni" and any(alias.name == "clients" for alias in node.names): + return True + return False + + +def test_api_server_does_not_import_client_package() -> None: + api_server = _REPO_ROOT / "vllm_omni/entrypoints/openai/api_server.py" + assert not _imports_clients_package(api_server) + + +def test_duplex_serving_stack_does_not_import_client_package() -> None: + duplex_pkg = _REPO_ROOT / "vllm_omni/entrypoints/duplex" + offenders = [path.name for path in sorted(duplex_pkg.glob("*.py")) if _imports_clients_package(path)] + assert not offenders, f"duplex serving modules import the client package: {offenders}" diff --git a/tests/entrypoints/openai_api/test_duplex_handler.py b/tests/entrypoints/openai_api/test_duplex_handler.py index c75f1e599..c69bc5c47 100644 --- a/tests/entrypoints/openai_api/test_duplex_handler.py +++ b/tests/entrypoints/openai_api/test_duplex_handler.py @@ -14,46 +14,59 @@ import numpy as np import pytest from starlette.websockets import WebSocketDisconnect +# Import the OpenAI entrypoints package at collection time, before pytest +# attaches its caplog handlers to the root logger. vllm's api_server +# transitively imports model_hosting_container_standards, whose import-time +# configure_root_logger() raises the level of every handler then attached to +# the root logger to ERROR. The serving runtime under test reaches this chain +# lazily (runtime_bridge -> entrypoints.openai.protocol.audio), so without +# this eager import the side effect fires mid-test and silences caplog for +# the remainder of the session. Before the capability predicate moved to +# vllm_omni.entrypoints.duplex.capability, this file imported the package at +# module scope implicitly. +import vllm_omni.entrypoints.openai # noqa: F401 from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig -from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlRequestError -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id -from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence, DuplexSessionLifecycleMessage -from vllm_omni.experimental.fullduplex.minicpmo45 import ( - MiniCPMO45NativeDuplexServingAdapter, - MiniCPMO45PcmAppendBuffer, -) -from vllm_omni.experimental.fullduplex.minicpmo45.data_plane import ( - MiniCPMO45DataPlaneContext, - MiniCPMO45DataPlaneSession, -) -from vllm_omni.experimental.fullduplex.minicpmo45.runtime import ( - MiniCPMO45DuplexRuntimeExtension, -) -from vllm_omni.experimental.fullduplex.minicpmo45.session import ( - MiniCPMO45ServingSessionState, -) -from vllm_omni.experimental.fullduplex.openai import vad as realtime_vad -from vllm_omni.experimental.fullduplex.openai.protocol import ( - DuplexCapabilities, +from vllm_omni.engine.duplex.control_client import DuplexControlRequestError +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity +from vllm_omni.engine.duplex.messages import DuplexFence, DuplexSessionLifecycleMessage +from vllm_omni.engine.duplex.runtime import duplex_resource_request_id +from vllm_omni.entrypoints.duplex import vad as realtime_vad +from vllm_omni.entrypoints.duplex.protocol import ( DuplexOverlapPolicy, DuplexPlaybackCommitPolicy, DuplexSession, DuplexSessionConfig, ResponseCreateOptions, ) -from vllm_omni.experimental.fullduplex.openai.realtime_session import NativeRealtimeSessionProtocol -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ServingRuntimeConfigError -from vllm_omni.experimental.fullduplex.openai.serving import ( +from vllm_omni.entrypoints.duplex.realtime_session import NativeRealtimeSessionProtocol +from vllm_omni.entrypoints.duplex.runtime_adapter import ServingRuntimeConfigError +from vllm_omni.entrypoints.duplex.serving import ( OmniDuplexSessionHandler, should_enable_duplex_endpoint, ) -from vllm_omni.experimental.fullduplex.openai.websocket import DuplexWebSocketActor -from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision -from vllm_omni.experimental.fullduplex.personaplex.serving_adapter import ( +from vllm_omni.entrypoints.duplex.websocket import DuplexWebSocketActor +from vllm_omni.model_executor.models.minicpmo_4_5.duplex import ( + MiniCPMO45NativeDuplexServingAdapter, + MiniCPMO45PcmAppendBuffer, +) +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import ( + minicpmo45_native_capabilities, +) +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.data_plane import ( + MiniCPMO45DataPlaneContext, + MiniCPMO45DataPlaneSession, +) +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import ( + MiniCPMO45DuplexRuntimeExtension, +) +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.session import ( + MiniCPMO45ServingSessionState, +) +from vllm_omni.model_executor.models.personaplex.duplex.serving_adapter import ( PersonaPlexServingRuntimeAdapter, ) from vllm_omni.outputs import OmniRequestOutput +from vllm_omni.outputs.duplex import attach_duplex_output_decision pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -62,12 +75,12 @@ def test_native_input_append_supports_explicit_session_opt_out(): session = DuplexSession( "sid-explicit-tts", DuplexSessionConfig(model="test-model"), - capabilities=DuplexCapabilities.minicpmo45_native(), + capabilities=minicpmo45_native_capabilities(), ) assert OmniDuplexSessionHandler._uses_native_input_append(session) is True - session.config.extra_body["minicpmo45_native_duplex"] = False + session.config.extra_body["native_duplex"] = False assert OmniDuplexSessionHandler._uses_native_input_append(session) is False @@ -293,7 +306,7 @@ class FakeEngineClient: class FakeChatService: duplex_serving_adapter_path = ( - "vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter" + "vllm_omni.model_executor.models.minicpmo_4_5.duplex.serving_adapter.MiniCPMO45ServingRuntimeAdapter" ) def __init__(self, engine_client: FakeEngineClient) -> None: @@ -844,7 +857,7 @@ def test_native_response_options_ignore_private_runtime_config(): session = DuplexSession( session_id="sid-response-private-config", config=DuplexSessionConfig(), - capabilities=DuplexCapabilities.minicpmo45_native(), + capabilities=minicpmo45_native_capabilities(), ) handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) @@ -878,7 +891,7 @@ def test_native_response_create_rejects_unwired_generation_options(payload): session = DuplexSession( session_id="sid-response-unwired-options", config=DuplexSessionConfig(), - capabilities=DuplexCapabilities.minicpmo45_native(), + capabilities=minicpmo45_native_capabilities(), ) handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient())) @@ -1244,7 +1257,7 @@ async def test_minicpmo_native_session_update_rejects_native_duplex_mode_flip(): { "type": "turn.signal", "event": "session.update", - "payload": {"extra_body": {"minicpmo45_native_duplex": False}}, + "payload": {"extra_body": {"native_duplex": False}}, } ) ws.put( @@ -1581,7 +1594,7 @@ def _native_session_create( event["session"]["model"] = "openbmb/MiniCPM-o-4_5" event["session"]["modalities"] = list(modalities or ["text"]) event["session"]["instructions"] = "You are a concise assistant." - event["session"]["extra_body"] = {"minicpmo45_native_duplex": True} + event["session"]["extra_body"] = {"native_duplex": True} return event @@ -1598,7 +1611,7 @@ def _native_realtime_session_update( "modalities": list(modalities or ["text"]), "instructions": "You are a concise assistant.", "idle_timeout_s": 1, - "extra_body": {"minicpmo45_native_duplex": True}, + "extra_body": {"native_duplex": True}, }, } @@ -1674,7 +1687,7 @@ def _install_direct_silence_scheduler( handler: OmniDuplexSessionHandler, session: DuplexSession, ) -> None: - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) async def _schedule(payload: object, **kwargs: Any) -> bool: if handler._native_silence_continuation_is_stale( @@ -1819,7 +1832,7 @@ def test_legacy_auto_overlap_policy_falls_back_to_listen_only(): overlap_policy=DuplexSessionConfig._normalize_overlap_policy("auto"), ), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() decision = handler._overlap_decision( session, @@ -2066,11 +2079,11 @@ def test_auto_response_post_response_silence_remains_model_owned_input(): @pytest.mark.asyncio async def test_auto_response_terminal_advances_open_realtime_input_without_commit(): handler, session = _auto_response_context("sid-auto-realtime-open-input", playback_active=True) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() response_id = session.active_response_id assert response_id is not None - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) native.input_since_commit = True native.speech_since_commit = True session.accumulate_overlap_speech(800) @@ -2113,11 +2126,11 @@ async def test_auto_response_terminal_allows_next_model_unit_while_playback_drai "sid-auto-realtime-open-before-overlap", playback_active=True, ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() response_id = session.active_response_id assert response_id is not None - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) native.input_since_commit = True native.speech_since_commit = True session.end_response(commit_text=False, preserve_request=True) @@ -2147,11 +2160,11 @@ async def test_auto_response_terminal_allows_next_model_unit_while_playback_drai @pytest.mark.asyncio async def test_realtime_committed_overlap_promotes_after_response_done(): handler, session = _auto_response_context("sid-auto-realtime-committed-input", playback_active=True) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() response_id = session.active_response_id assert response_id is not None - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) native.committed_audio_payload = _native_audio_payload() native.deferred_response_create = True session.accumulate_overlap_speech(1000) @@ -3061,7 +3074,7 @@ async def test_native_append_propagates_current_turn_fence_to_engine(): session_id="sid-fenced-append", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.turn_id = 2 session.begin_response(turn_id=2) ws = TimedWebSocket() @@ -3091,7 +3104,7 @@ async def test_minicpmo_auto_response_tts_segment_boundary_appends_silence_unit( session_id="sid-segment-boundary", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.begin_response(turn_id=0) session.turn_id = 1 session.bind_request(request_id) @@ -3140,7 +3153,7 @@ async def test_minicpmo_auto_response_pre_response_tts_boundary_continues_model_ session_id="sid-pre-response-boundary", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.bind_request(request_id) _install_direct_silence_scheduler(handler, session) ws = TimedWebSocket() @@ -3187,11 +3200,11 @@ async def test_minicpmo_pre_response_continuation_drops_after_model_turn_ends(): session_id="sid-stale-pre-response-boundary", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.bind_request(request_id) model_turn_id = session.turn_id - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) async def _stale_before_append(payload: object, **kwargs: Any) -> bool: session.complete_model_turn(model_turn_id) @@ -3237,7 +3250,7 @@ async def test_minicpmo_auto_response_continuation_stops_at_large_safety_boundar session_id="sid-long-response", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.begin_response(turn_id=0) session.turn_id = 1 session.bind_request(request_id) @@ -3283,10 +3296,10 @@ async def test_minicpmo_auto_response_boundary_listen_closes_response(): session_id="sid-boundary-listen", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() response_id = session.begin_response(turn_id=0) session.bind_request(request_id) - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) native.continuation_owner_id = f"response:{response_id}" native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS _install_direct_silence_scheduler(handler, session) @@ -3313,6 +3326,11 @@ async def test_minicpmo_auto_response_boundary_listen_closes_response(): assert emitted is True assert engine.appended == [] assert ws.sent_types() == ["response.listen", "response.done"] + # The listen terminates the precreated response: clients demultiplex the + # decision by its id, so the payload must carry the response identity. + listen_payload, done_payload = ws.sent[-2], ws.sent[-1] + assert listen_payload["response_id"] == response_id + assert done_payload["response_id"] == response_id assert session.active_response_id is None assert session.active_request_id == request_id assert session.turn_id == 1 @@ -3327,10 +3345,10 @@ async def test_minicpmo_auto_response_boundary_allows_next_model_turn_response() session_id="sid-boundary-next-turn", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.begin_response(turn_id=0) session.bind_request(request_id) - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) native.continuation_owner_id = f"response:{session.active_response_id}" native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS ws = TimedWebSocket() @@ -3382,10 +3400,10 @@ async def test_minicpmo_auto_response_boundary_does_not_close_new_epoch_response session_id="sid-boundary-race", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() old_response_id = session.begin_response(turn_id=0) session.bind_request(request_id) - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) native.continuation_owner_id = f"response:{old_response_id}" native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS sent: list[dict[str, Any]] = [] @@ -3418,7 +3436,7 @@ async def test_minicpmo_auto_response_pre_speak_listen_continues_same_response() session_id="sid-pre-speak-listen", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() response_id = session.begin_response(turn_id=0) session.turn_id = 1 session.bind_request(request_id) @@ -3458,7 +3476,7 @@ async def test_minicpmo_auto_response_listen_before_response_keeps_resumable_req session_id="sid-listen-before-response", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.bind_request(request_id) ws = TimedWebSocket() @@ -3482,7 +3500,7 @@ async def test_minicpmo_auto_response_listen_before_response_keeps_resumable_req assert close_reason is None assert emitted is True assert session.active_request_id == request_id - assert not handler._minicpmo_data_plane.is_terminal(request_id) + assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id) assert ws.sent_types().count("response.listen") == 1 @@ -3666,7 +3684,7 @@ async def test_minicpmo_auto_response_post_speak_listen_continues_same_response( session_id="sid-post-speak-listen", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() response_id = session.begin_response(turn_id=0) session.turn_id = 1 session.bind_request(request_id) @@ -3707,7 +3725,7 @@ async def test_minicpmo_auto_response_turn_end_preserves_resumable_request(): session_id="sid-turn-end-preserve-request", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.begin_response(turn_id=0) session.bind_request(request_id) ws = TimedWebSocket() @@ -3746,7 +3764,7 @@ async def test_minicpmo_auto_response_empty_turn_end_emits_model_listen(): session_id="sid-empty-turn-end", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.bind_request(request_id) ws = TimedWebSocket() @@ -3789,7 +3807,7 @@ async def test_minicpmo_auto_response_drops_late_audio_from_completed_model_turn session_id="sid-late-completed-turn", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.bind_request(request_id) session.complete_model_turn(0) ws = TimedWebSocket() @@ -4015,7 +4033,7 @@ async def test_duplex_auto_response_empty_terminal_does_not_create_empty_respons ) native_results = list( - handler._minicpmo_data_plane.project( + handler._serving_runtime_adapter.data_plane.project( {"data_plane_outputs": [output]}, context=_data_plane_context(session), ) @@ -4030,7 +4048,7 @@ async def test_duplex_auto_response_empty_terminal_does_not_create_empty_respons assert "response.created" not in [m.get("type") for m in sent] assert "response.done" not in [m.get("type") for m in sent] assert session.turn_id == 1 - assert not handler._minicpmo_data_plane.is_terminal(request_id) + assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id) @pytest.mark.asyncio @@ -4040,11 +4058,11 @@ async def test_minicpmo_native_duplex_session_close_cleans_auto_response_data_pl engine = FakeEngineClient() chat_service = FakeChatService(engine) handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) - handler._minicpmo_data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32)) - handler._minicpmo_data_plane.segment_text_delta(request_id, "hello") - handler._minicpmo_data_plane.mark_terminal(request_id) + handler._serving_runtime_adapter.data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32)) + handler._serving_runtime_adapter.data_plane.segment_text_delta(request_id, "hello") + handler._serving_runtime_adapter.data_plane.mark_terminal(request_id) native = MiniCPMO45ServingSessionState() - handler._minicpmo_sessions[session_id] = native + handler._serving_runtime_adapter.session_states[session_id] = native event = _native_session_create(session_id) event["session"]["extra_body"]["auto_response"] = True @@ -4054,8 +4072,8 @@ async def test_minicpmo_native_duplex_session_close_cleans_auto_response_data_pl await handler.handle_session(ws) - assert not handler._minicpmo_data_plane.has_request(request_id) - assert session_id not in handler._minicpmo_sessions + assert not handler._serving_runtime_adapter.data_plane.has_request(request_id) + assert session_id not in handler._serving_runtime_adapter.session_states @pytest.mark.asyncio @@ -4923,7 +4941,7 @@ async def test_realtime_disconnect_detaches_resumable_session_without_runtime_cl ("sid-resumable-disconnect", DuplexLeaseActivity.DETACH), ] assert handler._registry.get("sid-resumable-disconnect") is not None - assert "sid-resumable-disconnect" in handler._minicpmo_sessions + assert "sid-resumable-disconnect" in handler._serving_runtime_adapter.session_states @pytest.mark.asyncio @@ -4947,7 +4965,7 @@ async def test_realtime_initial_resume_token_delivery_failure_closes_unrecoverab assert engine.closed == [("sid-created-delivery-failure", "disconnect")] assert handler._registry.get("sid-created-delivery-failure") is None - assert "sid-created-delivery-failure" not in handler._minicpmo_sessions + assert "sid-created-delivery-failure" not in handler._serving_runtime_adapter.session_states @pytest.mark.asyncio @@ -4983,7 +5001,7 @@ async def test_realtime_send_after_created_disconnect_preserves_resumable_sessio ("sid-created-then-disconnected", DuplexLeaseActivity.DETACH), ] assert handler._registry.get("sid-created-then-disconnected") is not None - assert "sid-created-then-disconnected" in handler._minicpmo_sessions + assert "sid-created-then-disconnected" in handler._serving_runtime_adapter.session_states @pytest.mark.asyncio @@ -5007,7 +5025,7 @@ async def test_realtime_resume_rotates_token_replays_and_preserves_runtime_ident second = TimedWebSocket(receive_timeout_s=0.1) second.query_params = { "model": "openbmb/MiniCPM-o-4_5", - "minicpmo45_native_duplex": "1", + "native_duplex": "1", "resume": "1", } second.put( @@ -5083,7 +5101,7 @@ async def test_realtime_resume_preserves_append_tail_order_across_connections(): assert created is not None session = handler._registry.get("sid-resume-append-order") assert session is not None - native_state = handler._minicpmo_session_state(session) + native_state = handler._runtime_session_state(session) first_scheduler = native_state.silence_continuation_scheduler assert first_scheduler is not None first.put( @@ -5101,7 +5119,7 @@ async def test_realtime_resume_preserves_append_tail_order_across_connections(): second = TimedWebSocket(receive_timeout_s=0.5) second.query_params = { "model": "openbmb/MiniCPM-o-4_5", - "minicpmo45_native_duplex": "1", + "native_duplex": "1", "resume": "1", } second.put( @@ -5402,7 +5420,7 @@ async def test_realtime_disconnect_grace_cancels_only_orphan_response(): assert engine.aborted == ["orphan-request"] assert orphan_task.cancelled() assert session.active_response_id is None - assert "sid-grace-response" in handler._minicpmo_sessions + assert "sid-grace-response" in handler._serving_runtime_adapter.session_states @pytest.mark.asyncio @@ -5453,7 +5471,7 @@ async def test_realtime_idle_ttl_lifecycle_removes_detached_serving_projection() await asyncio.sleep(0.05) assert handler._registry.get("sid-idle-expired") is None - assert "sid-idle-expired" not in handler._minicpmo_sessions + assert "sid-idle-expired" not in handler._serving_runtime_adapter.session_states assert "sid-idle-expired" not in handler._session_tasks assert engine.closed == [] @@ -6428,7 +6446,7 @@ async def test_minicpmo_native_duplex_separates_public_and_runtime_config(monkey return [0.25] * 1600, 16000 monkeypatch.setattr( - "vllm_omni.experimental.fullduplex.minicpmo45.MiniCPMO45NativeDuplexServingAdapter.resolve_ref_audio", + "vllm_omni.model_executor.models.minicpmo_4_5.duplex.MiniCPMO45NativeDuplexServingAdapter.resolve_ref_audio", fake_resolve_ref_audio, ) event = _native_session_create("sid-native-ref-audio", modalities=["text", "audio"]) @@ -6554,7 +6572,7 @@ async def test_minicpmo_native_duplex_preserves_ref_audio_channels_until_normali return np.tile(np.array([[0.25, -0.25], [0.5, -0.5]], dtype=np.float32), (800, 1)), 16000 monkeypatch.setattr( - "vllm_omni.experimental.fullduplex.minicpmo45.adapter.MediaConnector", + "vllm_omni.model_executor.models.minicpmo_4_5.duplex.adapter.MediaConnector", FakeMediaConnector, ) @@ -6580,7 +6598,7 @@ async def test_minicpmo_native_duplex_rejects_ref_audio_path(): handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1) event = _native_session_create("sid-native-ref-path") event["session"]["extra_body"] = { - "minicpmo45_native_duplex": True, + "native_duplex": True, "ref_audio_path": "/tmp/ref.wav", } ws = TimedWebSocket() @@ -6603,7 +6621,7 @@ def test_minicpmo_native_duplex_explicit_barge_in_request_interrupts(): session = DuplexSession( session_id="sid-native-explicit-barge-disabled", config=DuplexSessionConfig(overlap_policy=DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value), - capabilities=DuplexCapabilities.minicpmo45_native(), + capabilities=minicpmo45_native_capabilities(), ) payload = { "type": "audio", @@ -6767,9 +6785,9 @@ async def test_minicpmo_auto_response_restarts_drain_when_append_races_idle_exit session_id="sid-native-late-output", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() session.bind_request(request_id) - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) projected_batches: list[object] = [] projected = asyncio.Event() @@ -7521,7 +7539,7 @@ async def test_minicpmo_native_auto_response_preserves_silence_continuations_acr session.capabilities.chunk_period_ms = 50 session.bind_request(request_id) response_id = session.begin_response(turn_id=session.turn_id) - native = handler._minicpmo_session_state(session) + native = handler._runtime_session_state(session) scheduler = native.silence_continuation_scheduler assert scheduler is not None @@ -7598,7 +7616,7 @@ async def test_minicpmo_native_duplex_uses_segment_text_metadata_for_transcript_ async def emit(output: object) -> None: result = {"data_plane_outputs": [output]} - for native_result in handler._minicpmo_data_plane.project( + for native_result in handler._serving_runtime_adapter.data_plane.project( result, context=_data_plane_context(session), ): @@ -7682,7 +7700,7 @@ async def test_minicpmo_native_duplex_continuous_speak_reuses_active_response_un async def emit(output: object) -> None: result = {"data_plane_outputs": [output]} - for native_result in handler._minicpmo_data_plane.project( + for native_result in handler._serving_runtime_adapter.data_plane.project( result, context=_data_plane_context(session), ): @@ -7932,7 +7950,7 @@ async def test_minicpmo_auto_response_segment_complete_continues_until_model_lis session_id="sid-native-segment-complete", config=DuplexSessionConfig(extra_body={"auto_response": True}), ) - session.capabilities = DuplexCapabilities.minicpmo45_native() + session.capabilities = minicpmo45_native_capabilities() response_id = session.begin_response() request_id = "duplex-sid-native-segment-complete-stage0" session.bind_request(request_id) @@ -7968,7 +7986,7 @@ async def test_minicpmo_auto_response_segment_complete_continues_until_model_lis assert emitted is False assert session.active_response_id == response_id assert session.active_request_id == request_id - assert not handler._minicpmo_data_plane.is_terminal(request_id) + assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id) assert sent == [] assert len(engine.appended) == 1 _, mode, payload, final = engine.appended[0] diff --git a/tests/entrypoints/test_async_omni_duplex.py b/tests/entrypoints/test_async_omni_duplex.py index 46bf668b3..81e0d75cb 100644 --- a/tests/entrypoints/test_async_omni_duplex.py +++ b/tests/entrypoints/test_async_omni_duplex.py @@ -1,21 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from types import SimpleNamespace import pytest -from vllm_omni.engine.messages import OutputMessage -from vllm_omni.entrypoints.async_omni import AsyncOmni -from vllm_omni.entrypoints.client_request_state import ClientRequestState -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import ( DuplexOutputAction, DuplexOutputDecision, duplex_resource_request_id, ) -from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision -from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient +from vllm_omni.engine.messages import OutputMessage +from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.entrypoints.client_request_state import ClientRequestState +from vllm_omni.entrypoints.duplex_request_client import DuplexRequestClient from vllm_omni.metrics.stats import OrchestratorAggregator, StageRequestStats, StageStats from vllm_omni.outputs import OmniRequestOutput +from vllm_omni.outputs.duplex import attach_duplex_output_decision def test_async_omni_uses_extracted_duplex_request_client(): diff --git a/tests/entrypoints/test_duplex_fence_propagation.py b/tests/entrypoints/test_duplex_fence_propagation.py index f7b8cdcfc..2f1292ec7 100644 --- a/tests/entrypoints/test_duplex_fence_propagation.py +++ b/tests/entrypoints/test_duplex_fence_propagation.py @@ -1,10 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import pytest from tests.entrypoints.openai_api.test_duplex_handler import FakeChatService, TimedWebSocket +from vllm_omni.engine.duplex.messages import DuplexFence from vllm_omni.entrypoints.async_omni import AsyncOmni -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.openai.protocol import DuplexSession, DuplexSessionConfig -from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler +from vllm_omni.entrypoints.duplex.protocol import DuplexSession, DuplexSessionConfig +from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/examples/test_minicpmo_realtime_duplex_simple_demo.py b/tests/examples/test_minicpmo_realtime_duplex_simple_demo.py index dfb7b32b1..d4a70e1b5 100644 --- a/tests/examples/test_minicpmo_realtime_duplex_simple_demo.py +++ b/tests/examples/test_minicpmo_realtime_duplex_simple_demo.py @@ -175,21 +175,26 @@ def test_open_streaming_response_requires_post_commit_drain(): def test_final_input_commit_checkpoints_playback_first(): demo = _load_demo_module() - client = demo.RealtimeDuplexClient("ws://unused") - client.events.add( + collector = demo.EventCollector() + collector.add( { "type": "response.created", "response": {"id": "resp-zero-audio"}, } ) - calls = [] + calls: list[dict[str, object]] = [] - async def send(event): - calls.append(event) + class _Client: + async def ack_playback(self, played_ms, *, response_id=None, item_id=None, committed_ms=None): + calls.append({"type": "playback.ack", "response_id": response_id, "played_ms": int(played_ms)}) - client.send = send - commit_sent_at_s = asyncio.run(demo._commit_input_after_playback_checkpoint(client)) + async def commit(self): + calls.append({"type": "input_audio_buffer.commit"}) + commit_sent_at_s = asyncio.run(demo._commit_input_after_playback_checkpoint(_Client(), collector)) + + # The zero-audio response is still acked (0 ms) before the commit: the + # ack checkpoints its history position on the server. assert [event["type"] for event in calls] == ["playback.ack", "input_audio_buffer.commit"] assert calls[0]["response_id"] == "resp-zero-audio" assert calls[0]["played_ms"] == 0 diff --git a/tests/examples/test_minicpmo_realtime_web_static.py b/tests/examples/test_minicpmo_realtime_web_static.py index 00859dabb..5fdab87d1 100644 --- a/tests/examples/test_minicpmo_realtime_web_static.py +++ b/tests/examples/test_minicpmo_realtime_web_static.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import re import shutil import subprocess @@ -33,7 +36,7 @@ def test_client_uses_proxy_relative_realtime_url_and_model_policy_session(): assert "new URL(config.realtimePath, window.location.href)" in source assert "url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'" in source assert "url.searchParams.set('autostart', '0')" in source - assert "url.searchParams.set('minicpmo45_native_duplex', '1')" in source + assert "url.searchParams.set('native_duplex', '1')" in source assert "auto_response: true" in source assert "input_audio_buffer.append" in source assert "input_audio_buffer.commit" not in source diff --git a/tests/model_executor/models/minicpmo_4_5/duplex/__init__.py b/tests/model_executor/models/minicpmo_4_5/duplex/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/e2e/features/fullduplex/minicpmo45/test_commit_policy.py b/tests/model_executor/models/minicpmo_4_5/duplex/test_commit_policy.py similarity index 94% rename from tests/e2e/features/fullduplex/minicpmo45/test_commit_policy.py rename to tests/model_executor/models/minicpmo_4_5/duplex/test_commit_policy.py index 5ee66b41a..b933f7173 100644 --- a/tests/e2e/features/fullduplex/minicpmo45/test_commit_policy.py +++ b/tests/model_executor/models/minicpmo_4_5/duplex/test_commit_policy.py @@ -1,8 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import pytest -from vllm_omni.experimental.fullduplex.openai.commit_policy import ( +from vllm_omni.entrypoints.duplex.commit_policy import ( CommitAction, CommitSnapshot, decide_commit_action, diff --git a/tests/e2e/features/fullduplex/minicpmo45/test_input.py b/tests/model_executor/models/minicpmo_4_5/duplex/test_input.py similarity index 97% rename from tests/e2e/features/fullduplex/minicpmo45/test_input.py rename to tests/model_executor/models/minicpmo_4_5/duplex/test_input.py index 5562fb237..4d6f3e46a 100644 --- a/tests/e2e/features/fullduplex/minicpmo45/test_input.py +++ b/tests/model_executor/models/minicpmo_4_5/duplex/test_input.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import base64 import numpy as np import pytest -from vllm_omni.experimental.fullduplex.minicpmo45.input import MiniCPMO45PcmAppendBuffer +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.input import MiniCPMO45PcmAppendBuffer pytestmark = [pytest.mark.core_model, pytest.mark.cpu] diff --git a/tests/model_executor/models/minicpmo_4_5/duplex/test_policy.py b/tests/model_executor/models/minicpmo_4_5/duplex/test_policy.py new file mode 100644 index 000000000..18468ef9c --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/duplex/test_policy.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +import pytest + +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import ( + MiniCPMO45DuplexPolicy, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_seed_tts_initial_text_is_part_of_native_duplex_context(): + prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts( + "Speak exactly.", + True, + "The quick brown fox.", + ) + + assert prefix == "<|im_start|>system\nSpeak exactly.\n<|audio_start|>" + assert suffix == ( + "<|audio_end|><|im_end|>\n<|im_start|>user\nThe quick brown fox.<|im_end|>\n<|im_start|>assistant\n" + ) diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index a05e4076f..450246696 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -61,7 +61,7 @@ class TestRegistryDeclaration: pipeline = OMNI_PIPELINES[_PIPELINE_KEY] assert pipeline.duplex_control_enabled is True assert pipeline.duplex_serving_adapter == ( - "vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter" + "vllm_omni.model_executor.models.minicpmo_4_5.duplex.serving_adapter.MiniCPMO45ServingRuntimeAdapter" ) assert not hasattr(pipeline, "max_native_duplex_sessions") diff --git a/tests/model_executor/models/nemotron_voicechat/__init__.py b/tests/model_executor/models/nemotron_voicechat/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/tests/model_executor/models/nemotron_voicechat/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/tests/model_executor/models/nemotron_voicechat/duplex/__init__.py b/tests/model_executor/models/nemotron_voicechat/duplex/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/tests/model_executor/models/nemotron_voicechat/duplex/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/tests/e2e/features/fullduplex/nemotron_voicechat/test_data_plane.py b/tests/model_executor/models/nemotron_voicechat/duplex/test_data_plane.py similarity index 91% rename from tests/e2e/features/fullduplex/nemotron_voicechat/test_data_plane.py rename to tests/model_executor/models/nemotron_voicechat/duplex/test_data_plane.py index 5790b11f1..4bdc835d9 100644 --- a/tests/e2e/features/fullduplex/nemotron_voicechat/test_data_plane.py +++ b/tests/model_executor/models/nemotron_voicechat/duplex/test_data_plane.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations from types import SimpleNamespace @@ -5,9 +8,9 @@ from types import SimpleNamespace import numpy as np import pytest -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.nemotron_voicechat.data_plane import ( +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import duplex_resource_request_id +from vllm_omni.model_executor.models.nemotron_voicechat.duplex.data_plane import ( NemotronVoiceChatDataPlaneContext, NemotronVoiceChatDataPlaneSession, ) diff --git a/tests/e2e/features/fullduplex/nemotron_voicechat/test_input.py b/tests/model_executor/models/nemotron_voicechat/duplex/test_input.py similarity index 92% rename from tests/e2e/features/fullduplex/nemotron_voicechat/test_input.py rename to tests/model_executor/models/nemotron_voicechat/duplex/test_input.py index 84c8b123a..0e8ff504c 100644 --- a/tests/e2e/features/fullduplex/nemotron_voicechat/test_input.py +++ b/tests/model_executor/models/nemotron_voicechat/duplex/test_input.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import base64 @@ -5,7 +8,7 @@ import base64 import numpy as np import pytest -from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import ( +from vllm_omni.model_executor.models.nemotron_voicechat.duplex.input import ( NemotronVoiceChatPcmAppendBuffer, ) diff --git a/tests/e2e/features/fullduplex/nemotron_voicechat/test_runtime_contract.py b/tests/model_executor/models/nemotron_voicechat/duplex/test_runtime_contract.py similarity index 92% rename from tests/e2e/features/fullduplex/nemotron_voicechat/test_runtime_contract.py rename to tests/model_executor/models/nemotron_voicechat/duplex/test_runtime_contract.py index 0b32322c0..6d2fbaada 100644 --- a/tests/e2e/features/fullduplex/nemotron_voicechat/test_runtime_contract.py +++ b/tests/model_executor/models/nemotron_voicechat/duplex/test_runtime_contract.py @@ -10,12 +10,12 @@ import numpy as np import pytest from vllm.sampling_params import SamplingParams -from vllm_omni.experimental.fullduplex.engine.contracts import DuplexInputMode -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.nemotron_voicechat.runtime import ( +from vllm_omni.engine.duplex.contracts import DuplexInputMode +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.model_executor.models.nemotron_voicechat.duplex.runtime import ( NemotronVoiceChatDuplexRuntimeExtension, ) -from vllm_omni.experimental.fullduplex.nemotron_voicechat.serving_adapter import ( +from vllm_omni.model_executor.models.nemotron_voicechat.duplex.serving_adapter import ( NemotronVoiceChatServingRuntimeAdapter, _render_tool_response, ) diff --git a/tests/model_executor/models/personaplex/__init__.py b/tests/model_executor/models/personaplex/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/tests/model_executor/models/personaplex/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/tests/model_executor/models/personaplex/duplex/__init__.py b/tests/model_executor/models/personaplex/duplex/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/tests/model_executor/models/personaplex/duplex/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/tests/e2e/features/fullduplex/personaplex/test_stage0_runtime.py b/tests/model_executor/models/personaplex/duplex/test_stage0_runtime.py similarity index 97% rename from tests/e2e/features/fullduplex/personaplex/test_stage0_runtime.py rename to tests/model_executor/models/personaplex/duplex/test_stage0_runtime.py index 3bdc4456b..b585399b7 100644 --- a/tests/e2e/features/fullduplex/personaplex/test_stage0_runtime.py +++ b/tests/model_executor/models/personaplex/duplex/test_stage0_runtime.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import base64 from types import SimpleNamespace @@ -8,11 +8,11 @@ import numpy as np import pytest import torch -from vllm_omni.experimental.fullduplex.personaplex.policy import ( +from vllm_omni.model_executor.models.personaplex.duplex.policy import ( SILENCE_TOKENS, SINE_TOKENS, ) -from vllm_omni.experimental.fullduplex.personaplex.stage0 import ( +from vllm_omni.model_executor.models.personaplex.duplex.stage0 import ( PersonaPlexStage0DuplexRuntime, ) from vllm_omni.model_executor.models.personaplex.personaplex_talker import ( diff --git a/tests/e2e/features/fullduplex/personaplex/test_unified_runtime.py b/tests/model_executor/models/personaplex/duplex/test_unified_runtime.py similarity index 91% rename from tests/e2e/features/fullduplex/personaplex/test_unified_runtime.py rename to tests/model_executor/models/personaplex/duplex/test_unified_runtime.py index f2970a240..8d4c1b589 100644 --- a/tests/e2e/features/fullduplex/personaplex/test_unified_runtime.py +++ b/tests/model_executor/models/personaplex/duplex/test_unified_runtime.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import base64 from pathlib import Path @@ -14,25 +14,25 @@ from vllm_omni.config.stage_config import ( load_deploy_config, merge_pipeline_deploy, ) -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import ( DuplexInputMode, ) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( +from vllm_omni.entrypoints.duplex.runtime_adapter import ( ServingRuntimeConfigError, ) -from vllm_omni.experimental.fullduplex.personaplex.config import DEFAULT_PERSONA -from vllm_omni.experimental.fullduplex.personaplex.data_plane import ( +from vllm_omni.model_executor.models.personaplex.duplex.config import DEFAULT_PERSONA +from vllm_omni.model_executor.models.personaplex.duplex.data_plane import ( PersonaPlexDataPlaneContext, PersonaPlexDataPlaneSession, ) -from vllm_omni.experimental.fullduplex.personaplex.input import ( +from vllm_omni.model_executor.models.personaplex.duplex.input import ( PersonaPlexPcmAppendBuffer, ) -from vllm_omni.experimental.fullduplex.personaplex.runtime_extension import ( +from vllm_omni.model_executor.models.personaplex.duplex.runtime_extension import ( PersonaPlexDuplexRuntimeExtension, ) -from vllm_omni.experimental.fullduplex.personaplex.serving_adapter import ( +from vllm_omni.model_executor.models.personaplex.duplex.serving_adapter import ( PersonaPlexServingRuntimeAdapter, ) from vllm_omni.model_executor.models.personaplex.pipeline import ( @@ -184,11 +184,11 @@ def test_e2e_driver_rejects_unexpected_input_sha256(tmp_path: Path) -> None: [ ( "duplex_runtime_extension", - "vllm_omni.experimental.fullduplex.personaplex.runtime_extension.PersonaPlexDuplexRuntimeExtension", + "vllm_omni.model_executor.models.personaplex.duplex.runtime_extension.PersonaPlexDuplexRuntimeExtension", ), ( "duplex_serving_adapter", - "vllm_omni.experimental.fullduplex.personaplex.serving_adapter.PersonaPlexServingRuntimeAdapter", + "vllm_omni.model_executor.models.personaplex.duplex.serving_adapter.PersonaPlexServingRuntimeAdapter", ), ], ) @@ -286,9 +286,14 @@ def test_personaplex_runtime_config_update_rejects_changed_voice() -> None: assert unchanged["personaplex_voice_prompt"] == "NATF2.pt" -def test_personaplex_rejects_minicpmo_only_opt_out_flag() -> None: +def test_personaplex_rejects_native_duplex_opt_out_flag() -> None: + # PersonaPlex is always model-native; the per-session opt-in knob is + # server-owned there, under both the canonical name and its deprecated + # model-prefixed alias. adapter = PersonaPlexServingRuntimeAdapter(lambda *_: None) + with pytest.raises(ServingRuntimeConfigError, match="native_duplex"): + adapter.validate_client_extra_body({"native_duplex": False}) with pytest.raises(ServingRuntimeConfigError, match="minicpmo45_native_duplex"): adapter.validate_client_extra_body({"minicpmo45_native_duplex": False}) diff --git a/tests/e2e/features/fullduplex/personaplex/test_streaming_code2wav.py b/tests/model_executor/models/personaplex/test_streaming_code2wav.py similarity index 99% rename from tests/e2e/features/fullduplex/personaplex/test_streaming_code2wav.py rename to tests/model_executor/models/personaplex/test_streaming_code2wav.py index abe2146b6..dd71d649c 100644 --- a/tests/e2e/features/fullduplex/personaplex/test_streaming_code2wav.py +++ b/tests/model_executor/models/personaplex/test_streaming_code2wav.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from types import SimpleNamespace diff --git a/tests/worker/test_native_duplex_hooks.py b/tests/worker/test_native_duplex_hooks.py index a30dcdb58..c47802fbc 100644 --- a/tests/worker/test_native_duplex_hooks.py +++ b/tests/worker/test_native_duplex_hooks.py @@ -18,7 +18,7 @@ def _minicpmo_duplex_policy_case( state: SimpleNamespace, payload: dict[str, object], ): - from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow + from vllm_omni.model_executor.duplex_sampling import DuplexSamplingRow from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( MiniCPMO45OmniForConditionalGeneration, ) @@ -45,7 +45,7 @@ def _minicpmo_duplex_policy_case( def test_minicpmo_model_hook_owns_duplex_sampling_rows_and_force_listen(): - from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow + from vllm_omni.model_executor.duplex_sampling import DuplexSamplingRow from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( MiniCPMO45OmniForConditionalGeneration, ) @@ -329,7 +329,7 @@ def test_minicpmo_model_hook_ignores_serving_new_user_turn_marker(): def test_generic_ar_runner_builds_typed_duplex_sampling_rows(): - from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingHelper + from vllm_omni.model_executor.duplex_sampling import DuplexSamplingHelper from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner runner = GPUARModelRunner.__new__(GPUARModelRunner) @@ -532,7 +532,7 @@ def test_minicpmo_stage0_routes_duplex_metadata_per_batched_request(): def test_minicpmo_stage0_rejects_invalid_resolved_ref_audio(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -546,7 +546,7 @@ def test_minicpmo_stage0_rejects_invalid_resolved_ref_audio(): def test_minicpmo_stage0_special_token_ids_are_tokenizer_derived(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -583,7 +583,7 @@ def test_minicpmo_stage0_special_token_ids_are_tokenizer_derived(): def test_minicpmo_stage0_rejects_unknown_special_token_fallbacks(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -659,7 +659,7 @@ def _transformers_processor_stub(auto_processor): def test_minicpmo_stage0_loads_processor_from_hf_id(monkeypatch): import sys - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -701,7 +701,7 @@ def test_minicpmo_stage0_loads_processor_from_hf_id(monkeypatch): def test_minicpmo_stage0_processor_load_preserves_original_exception(monkeypatch): import sys - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -728,7 +728,7 @@ def test_minicpmo_stage0_processor_load_preserves_original_exception(monkeypatch def test_minicpmo_stage0_processor_requires_tokenizer(monkeypatch): import sys - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -751,7 +751,7 @@ def test_minicpmo_stage0_processor_requires_tokenizer(monkeypatch): def test_minicpmo_stage0_loaded_processor_validates_special_tokens(monkeypatch): import sys - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -793,7 +793,7 @@ def test_minicpmo_stage0_loaded_processor_validates_special_tokens(monkeypatch): def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -855,7 +855,7 @@ def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format(): def _stage0_vision_runtime(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -912,7 +912,7 @@ def test_minicpmo_stage0_puts_every_frame_of_an_append_in_one_unit(): that closed a single unit left a frame over, failed the prefill, and still advanced the session state. """ - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( _MiniCPMO45Stage0SessionState, ) @@ -936,7 +936,7 @@ def test_minicpmo_stage0_wraps_hd_slices_after_the_source_image(): """Official stacked pair is max_slice_nums=[2, 1]: source + slices, then audio.""" import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( _MiniCPMO45Stage0SessionState, ) @@ -985,7 +985,7 @@ def test_minicpmo_stage0_reports_frames_that_close_no_unit_instead_of_dropping_t Frames are not carried across appends, so one arriving early used to vanish under a generic "audio not enough" buffering result. """ - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( _MiniCPMO45Stage0SessionState, ) @@ -1010,7 +1010,7 @@ def test_minicpmo_stage0_reports_frames_that_close_no_unit_instead_of_dropping_t def test_minicpmo_stage0_speech_append_sets_pending_context_once(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1118,7 +1118,7 @@ def test_minicpmo_stage0_speech_append_sets_pending_context_once(): def test_minicpmo_stage0_failed_append_does_not_set_pending_speech_context(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1173,7 +1173,7 @@ def test_minicpmo_stage0_failed_append_does_not_set_pending_speech_context(): def test_minicpmo_stage0_streaming_processor_is_isolated_per_session(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1220,7 +1220,7 @@ def test_minicpmo_stage0_streaming_processor_is_isolated_per_session(): def test_minicpmo_stage0_data_plane_next_append_reinjects_previous_listen(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1283,7 +1283,7 @@ def test_minicpmo_stage0_data_plane_next_append_reinjects_previous_listen(): def test_minicpmo_stage0_data_plane_turn_eos_closes_previous_unit(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1346,7 +1346,7 @@ def test_minicpmo_stage0_data_plane_turn_eos_closes_previous_unit(): def test_minicpmo_stage0_data_plane_model_owned_turn_boundary_preserves_audio_cache(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1411,7 +1411,7 @@ def test_minicpmo_stage0_data_plane_model_owned_turn_boundary_preserves_audio_ca def test_minicpmo_stage0_data_plane_final_first_chunk_does_not_add_silence_unit(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1485,7 +1485,7 @@ def test_minicpmo_stage0_data_plane_final_first_chunk_does_not_add_silence_unit( def test_minicpmo_stage0_final_does_not_promote_first_chunk_alignment_tail_to_unit(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1567,7 +1567,7 @@ def test_minicpmo_stage0_final_does_not_promote_first_chunk_alignment_tail_to_un def test_minicpmo_stage0_runtime_uses_loaded_vllm_embed_tokens_when_get_input_embeddings_is_broken(): import torch - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, ) @@ -1603,7 +1603,7 @@ def test_minicpmo_stage0_runtime_uses_loaded_vllm_embed_tokens_when_get_input_em def test_minicpmo_remote_config_patch_handles_nested_and_dict_configs(): - from vllm_omni.experimental.fullduplex.minicpmo45.compat import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.compat import ( patch_minicpmo_remote_config, ) @@ -1625,7 +1625,7 @@ def test_minicpmo_remote_config_patch_handles_nested_and_dict_configs(): def test_minicpmo_stage0_short_audio_buffers_without_context_mutation(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) @@ -1701,7 +1701,7 @@ def test_minicpmo_stage0_native_sampler_penalizes_repeated_text_token(): def test_minicpmo_stage0_native_sampler_penalizes_text_from_prior_chunk(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( _MiniCPMO45Stage0SessionState, ) from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( @@ -1761,7 +1761,7 @@ def test_minicpmo_stage0_native_sampler_penalizes_text_from_prior_chunk(): def test_minicpmo_stage0_native_sampler_matches_official_negative_logit_penalty(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( _MiniCPMO45Stage0SessionState, ) from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( @@ -1820,7 +1820,7 @@ def test_minicpmo_stage0_native_sampler_matches_official_negative_logit_penalty( def test_minicpmo_stage0_records_bounded_model_policy_history(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( _MiniCPMO45Stage0SessionState, ) from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( @@ -2248,7 +2248,7 @@ def test_minicpmo_stage0_native_sampler_preserves_model_chunk_eos(): def test_minicpmo_stage0_native_sampler_keeps_hard_chunk_cap(): - from vllm_omni.experimental.fullduplex.minicpmo45.policy import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import ( MiniCPMO45DuplexPolicy, ) from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( @@ -2360,7 +2360,7 @@ def test_minicpmo_stage0_native_sampler_cuts_before_request_length_cap(): def test_minicpmo_stage0_native_sampler_cuts_on_decoded_text_length(): - from vllm_omni.experimental.fullduplex.minicpmo45.policy import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import ( MiniCPMO45DuplexPolicy, ) from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( @@ -2598,7 +2598,7 @@ def test_minicpmo_stage0_native_sampler_uses_runner_duplex_rows(): def test_minicpmo_stage0_session_context_includes_resolved_ref_audio(): - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( MiniCPMO45Stage0DuplexRuntime, _MiniCPMO45Stage0SessionState, ) diff --git a/tools/pre_commit/check_forbidden_imports.py b/tools/pre_commit/check_forbidden_imports.py index 926e102fa..22ba693d4 100644 --- a/tools/pre_commit/check_forbidden_imports.py +++ b/tools/pre_commit/check_forbidden_imports.py @@ -79,19 +79,19 @@ CHECK_IMPORTS = { "vllm_omni/entrypoints/openai/video_stream_base.py", "vllm_omni/entrypoints/openai/video_stream_session.py", "vllm_omni/experimental/fullduplex/client.py", - "vllm_omni/experimental/fullduplex/engine/contracts.py", + "vllm_omni/engine/duplex/contracts.py", "vllm_omni/experimental/fullduplex/joyvl/memory/memory.py", - "vllm_omni/experimental/fullduplex/minicpmo45/adapter.py", - "vllm_omni/experimental/fullduplex/minicpmo45/input.py", - "vllm_omni/experimental/fullduplex/minicpmo45/runtime.py", - "vllm_omni/experimental/fullduplex/minicpmo45/stage0.py", - "vllm_omni/experimental/fullduplex/openai/audio.py", - "vllm_omni/experimental/fullduplex/openai/realtime_input.py", - "vllm_omni/experimental/fullduplex/openai/runtime_bridge.py", - "vllm_omni/experimental/fullduplex/openai/serving.py", - "vllm_omni/experimental/fullduplex/personaplex/input.py", - "vllm_omni/experimental/fullduplex/personaplex/runtime_extension.py", - "vllm_omni/experimental/fullduplex/personaplex/stage0.py", + "vllm_omni/model_executor/models/minicpmo_4_5/duplex/adapter.py", + "vllm_omni/model_executor/models/minicpmo_4_5/duplex/input.py", + "vllm_omni/model_executor/models/minicpmo_4_5/duplex/runtime.py", + "vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py", + "vllm_omni/entrypoints/duplex/audio.py", + "vllm_omni/entrypoints/duplex/realtime_input.py", + "vllm_omni/entrypoints/duplex/runtime_bridge.py", + "vllm_omni/entrypoints/duplex/serving.py", + "vllm_omni/model_executor/models/personaplex/duplex/input.py", + "vllm_omni/model_executor/models/personaplex/duplex/runtime_extension.py", + "vllm_omni/model_executor/models/personaplex/duplex/stage0.py", "vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py", "vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py", "vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_tokenizer.py", @@ -197,8 +197,6 @@ CHECK_IMPORTS = { "vllm_omni/entrypoints/openai/api_server.py", "vllm_omni/entrypoints/openai/serving_speech.py", "vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py", - "vllm_omni/experimental/fullduplex/personaplex/runtime.py", - "vllm_omni/experimental/fullduplex/personaplex/serving/server.py", "vllm_omni/model_executor/model_loader/weight_utils.py", "vllm_omni/model_executor/models/audex/checkpoint.py", "vllm_omni/model_executor/models/bagel/bagel.py", diff --git a/vllm_omni/benchmarks/omniinteract.py b/vllm_omni/benchmarks/omniinteract.py index 46822b486..b821d0235 100644 --- a/vllm_omni/benchmarks/omniinteract.py +++ b/vllm_omni/benchmarks/omniinteract.py @@ -22,7 +22,7 @@ from collections import Counter from collections.abc import Callable, Iterator, Sequence from dataclasses import dataclass, field from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urljoin, urlsplit import pybase64 as base64 @@ -33,24 +33,27 @@ from vllm_omni.benchmarks.data_modules.omniinteract_dataset import ( OmniInteractPreparedInput, case_manifest, ) -from vllm_omni.experimental.fullduplex.client import ( - PCM16_BYTES_PER_SAMPLE, - PCM16_SAMPLE_RATE, - RealtimeDuplexClient, - RealtimeEventCollector, - build_realtime_url, - chunk_period_ms, - has_residual_model_unit, - reference_audio_data_url, - summarize_session_request_metrics, - write_pcm16_wav, -) +# The duplex client library (vllm_omni.clients) is imported lazily inside the +# functions that need it: this module reaches the CLI import graph through the +# benchmark patch package (`vllm-omni serve` included), and the client package +# must stay out of that graph +# (tests/engine/test_duplex_import_boundary.py enforces the boundary). +if TYPE_CHECKING: + from vllm_omni.clients.duplex import EventCollector, WebSocketTransport + +# The duplex input wire format (16 kHz mono PCM16); mirrors +# vllm_omni.clients.duplex without importing it at module scope. +PCM16_SAMPLE_RATE = 16_000 +PCM16_BYTES_PER_SAMPLE = 2 OUTPUT_SAMPLE_RATE = 24_000 SUCCESS_ARTIFACTS = (".done", "output.wav", "wav_transcript.json", "events.json", "result.json") BATCH_ARTIFACTS = ("batch_summary.json", "official_eval_manifest.jsonl") ARTIFACT_LOCK_FILE = ".omniinteract.lock" _INPUT_CHUNK_MS = 200 +# Minimum new played audio between two incremental playback.ack sends for one +# response; completion acks always flush regardless of the step. +_ACK_STEP_MS = 200 VIDEO_FPS = 1.0 _COMPLETION_SETTLE_S = 2.0 logger = logging.getLogger(__name__) @@ -215,12 +218,13 @@ class _Playback: self.warnings: list[str] = [] self._total_samples: dict[str, int] = {} self._response_end_s: dict[str, float] = {} + self._acked_ms: dict[str, int] = {} def _warn_once(self, warning: str) -> None: if warning not in self.warnings: self.warnings.append(warning) - def ingest(self, events: RealtimeEventCollector) -> None: + def ingest(self, events: EventCollector) -> None: while self.cursor < len(events.events): index, self.cursor = self.cursor, self.cursor + 1 event = events.events[index] @@ -260,21 +264,52 @@ class _Playback: self._total_samples[response_id] = self._total_samples.get(response_id, 0) + samples self._response_end_s[response_id] = segment.end_s - async def acknowledge(self, client: RealtimeDuplexClient, now: float | None = None) -> None: + async def acknowledge(self, client: _RealtimeSession, now: float | None = None) -> None: + """Ack playback progress incrementally along the serialized clock. + + A real listener reports cumulative played audio while it plays, not + once after the fact. The first ack for a response goes out as soon as + its audio starts arriving, whatever the played amount: it checkpoints + the response's history position on the server, so a user input + committed later updates that slot in place instead of displacing it. + Later acks follow ``_ACK_STEP_MS``, with the exact total at drain. + """ events = client.events self.ingest(events) now = time.monotonic() if now is None else now - for response_id in self.completed - self.completion_acked: - total_samples = self._total_samples.get(response_id, 0) - if not total_samples or now < self._response_end_s[response_id]: + for response_id, total_samples in self._total_samples.items(): + if response_id in self.completion_acked: + continue + acked_ms = self._acked_ms.get(response_id) + if response_id in self.completed and now >= self._response_end_s[response_id]: + played_ms = total_samples * 1000 // OUTPUT_SAMPLE_RATE + if acked_ms is None or played_ms > acked_ms: + await client.send_playback_ack(response_id, played_ms) + self._acked_ms[response_id] = played_ms + self.completion_acked.add(response_id) + continue + played_ms = self._played_ms(response_id, now) + if acked_ms is not None and played_ms - acked_ms < _ACK_STEP_MS: continue - played_ms = total_samples * 1000 // OUTPUT_SAMPLE_RATE await client.send_playback_ack(response_id, played_ms) - self.completion_acked.add(response_id) + self._acked_ms[response_id] = played_ms + + def _played_ms(self, response_id: str, now: float) -> int: + """Cumulative audio of one response played by ``now`` on the serial clock.""" + played_samples = 0 + for segment in self.segments: + if segment.response_id != response_id or now <= segment.start_s: + continue + samples = len(segment.pcm16) // PCM16_BYTES_PER_SAMPLE + if now >= segment.end_s: + played_samples += samples + else: + played_samples += min(samples, int((now - segment.start_s) * OUTPUT_SAMPLE_RATE + 0.5)) + return played_samples * 1000 // OUTPUT_SAMPLE_RATE async def stream_inputs( - client: RealtimeDuplexClient, + client: _RealtimeSession, pcm: bytes, frames: Sequence[str | None], playback: _Playback, @@ -319,7 +354,7 @@ def _response_status(event: dict[str, object]) -> str | None: def response_ledger( - collector: RealtimeEventCollector, + collector: EventCollector, *, end: int | None = None, ) -> tuple[set[str], set[str]]: @@ -357,7 +392,7 @@ def _is_model_listen_decision(event: dict[str, object]) -> bool: def _has_post_commit_decision( - collector: RealtimeEventCollector, + collector: EventCollector, events: list[dict[str, object]], *, prior_response_ids: set[str], @@ -377,15 +412,34 @@ def _has_post_commit_decision( return False +# An ack computed just before a commit event was delivered can still lose the +# race server-side; the response it acks was already superseded, so the +# refusal is harmless — record it instead of failing the case. +_TOLERATED_ERROR_CODES = frozenset({"playback_ack_too_late"}) + + +def _error_code(event: dict[str, object]) -> str | None: + error = event.get("error") + code = error.get("code") if isinstance(error, dict) else event.get("code") + return code if isinstance(code, str) else None + + def _raise_if_session_terminated( - collector: RealtimeEventCollector, + collector: EventCollector, from_index: int, *, explicit_close_from: int | None = None, + warnings: list[str] | None = None, ) -> None: errors = collector.errors() - if errors: - raise RuntimeError(str(errors[-1])) + fatal = [event for event in errors if _error_code(event) not in _TOLERATED_ERROR_CODES] + if fatal: + raise RuntimeError(str(fatal[-1])) + if warnings is not None: + for event in errors: + warning = f"tolerated in-flight server rejection: {_error_code(event)}" + if warning not in warnings: + warnings.append(warning) for index, event in enumerate(collector.events[from_index:], start=from_index): event_type = event.get("type") if event_type not in {"session.expired", "session.closed"}: @@ -416,6 +470,8 @@ def _ensure_final_commit_tail(pcm: bytes, events: list[dict[str, object]]) -> by that correlation; the server pads the missing sample back to a full unit. """ + from vllm_omni.clients.duplex import chunk_period_ms, has_residual_model_unit + period_ms = chunk_period_ms(events) if len(pcm) >= PCM16_BYTES_PER_SAMPLE and not has_residual_model_unit(pcm, chunk_period_ms=period_ms): return pcm[:-PCM16_BYTES_PER_SAMPLE] @@ -423,7 +479,7 @@ def _ensure_final_commit_tail(pcm: bytes, events: list[dict[str, object]]) -> by async def wait_for_session_completion( - client: RealtimeDuplexClient, + client: _RealtimeSession, playback: _Playback, *, commit_from: int, @@ -438,7 +494,11 @@ async def wait_for_session_completion( stable_since = time.monotonic() while time.monotonic() < deadline: client.raise_if_reader_stopped() - _raise_if_session_terminated(client.events, commit_from if session_from is None else session_from) + _raise_if_session_terminated( + client.events, + commit_from if session_from is None else session_from, + warnings=playback.warnings, + ) await playback.acknowledge(client) if len(client.events.events) != last_event_count: last_event_count = len(client.events.events) @@ -522,6 +582,8 @@ def _atomic_write_json(path: Path, value: object) -> None: def _atomic_write_wav(path: Path, pcm: bytes | bytearray, rate: int) -> None: + from vllm_omni.clients.duplex import write_pcm16_wav + _atomic_replace( path, lambda temporary: write_pcm16_wav( @@ -578,7 +640,7 @@ def _publish_success_artifacts( def _collect_output( - collector: RealtimeEventCollector, + collector: EventCollector, playback: _Playback, *, stream_start: float, @@ -677,7 +739,7 @@ def _collect_output( def prepare_success_artifacts( root: Path, case: OmniInteractCase, - collector: RealtimeEventCollector, + collector: EventCollector, *, playback: _Playback | None = None, stream_start: float, @@ -770,6 +832,8 @@ def _websocket_url(config: OmniInteractBenchmarkConfig, session_id: str) -> str: if urlsplit(config.endpoint).scheme else urljoin(config.base_url.rstrip("/") + "/", config.endpoint.lstrip("/")) ) + from vllm_omni.clients.duplex import build_realtime_url + return build_realtime_url( endpoint, config.model, @@ -779,9 +843,121 @@ def _websocket_url(config: OmniInteractBenchmarkConfig, session_id: str) -> str: ) +class _RealtimeSession: + """The public :class:`DuplexClient` behind the benchmark's session surface. + + The case runner works in event-cursor style over one growing collector, + so this wrapper pairs a ``DuplexClient`` with an :class:`EventCollector` + fed by a background consumer task and exposes the few calls the runner + needs: raw sends, the final commit, cumulative playback acks, reader + liveness checks, and a confirmed close. + """ + + _MAX_FRAME_BYTES = 64 * 1024 * 1024 + + def __init__(self, config: OmniInteractBenchmarkConfig, session_id: str, reference_audio: str) -> None: + from vllm_omni.clients.duplex import DuplexClient, EventCollector, SessionConfig + + self.session_config = SessionConfig( + ref_audio=reference_audio, + overlap_policy="listen_only", + playback_commit_policy="ack_only", + idle_timeout_s=float(config.timeout_s), + extra_body={ + "native_duplex": True, + "force_listen_count": 0, + **(config.extra_body or {}), + }, + ) + self.url = _websocket_url(config, session_id) + headers = dict(config.extra_headers or {}) + + async def connect(url: str) -> WebSocketTransport: + import websockets + + return await websockets.connect( + url, + max_size=self._MAX_FRAME_BYTES, + additional_headers=headers or None, + ) + + self.events = EventCollector() + self._client = DuplexClient( + self.url, + model=config.model, + config=self.session_config, + session_id=session_id, + reconnect=None, + heartbeat_interval_s=None, + handshake_timeout_s=min(config.timeout_s, 20.0), + connect=connect, + ) + self._consume_task: asyncio.Task[None] | None = None + + async def __aenter__(self) -> _RealtimeSession: + await self._client.__aenter__() + self._consume_task = asyncio.create_task(self.events.consume(self._client)) + # The handshake consumed session.created before the collector + # subscribed; seed it so capability lookups and cursors still work. + self.events.add({"type": "session.created", "session": dict(self._client.session_info)}) + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + try: + await self._client.__aexit__(exc_type, exc, traceback) + finally: + if self._consume_task is not None: + self._consume_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._consume_task + + async def send(self, event: dict[str, object]) -> None: + await self._client.send(event) + + async def commit(self) -> None: + await self._client.commit() + + async def send_playback_ack(self, response_id: str, played_ms: int) -> None: + await self._client.ack_playback( + played_ms, + response_id=response_id, + item_id=f"item_{response_id}", + ) + + def raise_if_reader_stopped(self) -> None: + """Fail a caller waiting on events after the event consumer exits.""" + task = self._consume_task + if task is None or not task.done(): + return + if task.cancelled(): + raise ConnectionError("Realtime event consumer was cancelled") + error = task.exception() + if error is not None: + raise ConnectionError("Realtime session failed") from error + raise ConnectionError("Realtime session closed before the requested event arrived") + + async def close_session(self, *, timeout_s: float = 20.0) -> None: + await self._client.close(timeout_s=timeout_s) + if self._consume_task is not None: + # Let the collector drain session.closed before callers check it. + with contextlib.suppress(asyncio.TimeoutError, asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(self._consume_task), timeout=5.0) + + +def _audio_rtf_from_raw_metric(raw_metric: dict[str, object]) -> float | None: + """Derive the audio RTF from a client-reported raw request metric.""" + from vllm_omni.metrics.definitions import compute_audio_rtf + + generation_ms = raw_metric.get("audio_generation_ms") + duration_ms = raw_metric.get("audio_duration_ms") + if not isinstance(generation_ms, int | float) or not isinstance(duration_ms, int | float) or duration_ms <= 0: + return None + return round(compute_audio_rtf(float(generation_ms) / 1000.0, float(duration_ms) / 1000.0), 6) + + def _populate_response_metrics( result: OmniInteractCaseResult, - collector: RealtimeEventCollector, + collector: EventCollector, *, stream_start: float, ) -> None: @@ -807,6 +983,10 @@ def _populate_response_metrics( "response_id": response_id, **(raw_metric if isinstance(raw_metric, dict) else {}), } + if isinstance(raw_metric, dict): + # The client reports raw data only; the RTF is derived here with + # the canonical server-side metric definition. + metric["rtf"] = _audio_rtf_from_raw_metric(raw_metric) if isinstance(stage0, dict): metric["stage0_tokens"] = dict(stage0) output_tokens += int(stage0.get("output_token_count") or 0) @@ -814,6 +994,8 @@ def _populate_response_metrics( request_metrics.append(metric) result.output_tokens = output_tokens result.duplex_request_metrics = request_metrics + from vllm_omni.clients.duplex import summarize_session_request_metrics + result.duplex_session_metrics = summarize_session_request_metrics( request_metrics, session_id=result.session_id, @@ -847,6 +1029,8 @@ async def run_omniinteract_case( started_at = time.monotonic() try: if prepared_input is None: + from vllm_omni.clients.duplex import reference_audio_data_url + reference_audio = reference_audio_data_url(config.ref_audio) duration, pcm, frames = await asyncio.to_thread( prepare_media, @@ -862,18 +1046,8 @@ async def run_omniinteract_case( frames = prepared_input.video_frames if not any(frames): raise ValueError(f"No video frames were decoded from {case.video_path}") - async with RealtimeDuplexClient( - _websocket_url(config, session_id), additional_headers=config.extra_headers - ) as client: - session_from = len(client.events.events) - await client.configure( - config.model, - ref_audio=reference_audio, - session_id=session_id, - extra_body=config.extra_body, - idle_timeout_s=config.timeout_s, - timeout_s=min(config.timeout_s, 20.0), - ) + async with _RealtimeSession(config, session_id, reference_audio) as client: + session_from = 0 # the collector holds only this session's events pcm = _ensure_final_commit_tail(pcm, client.events.events) playback = _Playback() stream_start = time.monotonic() @@ -897,17 +1071,19 @@ async def run_omniinteract_case( settle_s=_COMPLETION_SETTLE_S, ) await playback.acknowledge(client, playback.end_s) - _raise_if_session_terminated(client.events, session_from) + _raise_if_session_terminated(client.events, session_from, warnings=playback.warnings) close_from = len(client.events.events) await client.close_session(timeout_s=min(config.timeout_s, 20.0)) except Exception: with contextlib.suppress(Exception): await client.close_session(timeout_s=min(config.timeout_s, 20.0)) raise - errors = client.events.errors() - if errors: - raise RuntimeError(str(errors[-1])) - _raise_if_session_terminated(client.events, session_from, explicit_close_from=close_from) + _raise_if_session_terminated( + client.events, + session_from, + explicit_close_from=close_from, + warnings=playback.warnings, + ) result.latency_s = time.monotonic() - started_at result.input_audio_chunks, result.input_video_frames = chunks, frame_count result.pacing_mean_lag_s, result.pacing_max_lag_s = mean_lag, max_lag diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index fd52c197b..b51dd2ee4 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -18,7 +18,7 @@ from collections.abc import Iterable from dataclasses import dataclass, replace from datetime import datetime from pathlib import Path -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal import aiohttp import numpy as np @@ -70,6 +70,7 @@ from vllm_omni.benchmarks.omniinteract import ( VIDEO_FPS, OmniInteractBenchmarkConfig, OmniInteractCaseResult, + _audio_rtf_from_raw_metric, clear_batch_artifacts, clear_case_artifacts, prepare_media, @@ -83,16 +84,18 @@ from vllm_omni.benchmarks.omniinteract import ( from vllm_omni.benchmarks.omniinteract import ( write_batch_artifacts as write_omniinteract_batch_artifacts, ) -from vllm_omni.experimental.fullduplex.client import ( - RealtimeDuplexClient, - build_realtime_url, - reference_audio_data_url, - summarize_session_request_metrics, - wait_for, -) from vllm_omni.metrics import definitions as defs from vllm_omni.metrics.utils import coerce_positive_int_scalar +if TYPE_CHECKING: + from vllm_omni.clients.duplex import DuplexClient + +# The duplex client library (vllm_omni.clients) is imported lazily inside the +# functions that drive Realtime sessions: this module is imported for its +# patching side effects by the CLI package (`vllm-omni serve` included), and +# the client package must stay out of that import graph +# (tests/engine/test_duplex_import_boundary.py enforces the boundary). + logger = init_logger(__name__) _AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S" @@ -423,6 +426,8 @@ def get_samples(args, tokenizer): ) if not requests: raise ValueError("No OmniInteract sessions were selected") + from vllm_omni.clients.duplex import reference_audio_data_url + encoded_ref_audio = reference_audio_data_url(options.ref_audio) assert encoded_ref_audio is not None for request in requests: @@ -1531,6 +1536,8 @@ async def async_request_openai_audio_speech( def _realtime_websocket_url(api_url: str) -> str: + from vllm_omni.clients.duplex import build_realtime_url + return build_realtime_url(api_url, None, native_duplex=None) @@ -1685,11 +1692,93 @@ async def _async_request_omniinteract( return output +class _RealtimeTTSProbe: + """Explicit-session Realtime TTS driver over the public duplex client. + + Owns an :class:`EventCollector` (``.events``) so turn metrics can be read + with controlled timestamps; the session handshake happens in + :meth:`configure` because the benchmark decides the session shape at + request time. + """ + + def __init__(self, url: str) -> None: + from vllm_omni.clients.duplex import EventCollector + + self._url = url + self.events: EventCollector = EventCollector() + self._client: DuplexClient | None = None + self._consume_task: asyncio.Task[None] | None = None + + async def __aenter__(self) -> "_RealtimeTTSProbe": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self._client is not None: + await self._client.__aexit__(exc_type, exc, tb) + if self._consume_task is not None: + try: + await asyncio.wait_for(self._consume_task, timeout=5.0) + except (asyncio.TimeoutError, asyncio.CancelledError): + self._consume_task.cancel() + + async def configure( + self, + model: str, + *, + output_audio_format: str = "pcm16", + instructions: str | None = None, + native_duplex: bool = False, + auto_response: bool = False, + extra_body: dict[str, object] | None = None, + session_id: str | None = None, + timeout_s: float = 120.0, + ) -> None: + from vllm_omni.clients.duplex import AudioFormat, DuplexClient, SessionConfig + + session_extra_body: dict[str, object] = dict(extra_body or {}) + session_extra_body["native_duplex"] = bool(native_duplex) + config = SessionConfig( + output_audio=AudioFormat(output_audio_format, 24_000), + instructions=instructions, + auto_response=auto_response, + overlap_policy="listen_only", + playback_commit_policy="ack_only", + extra_body=session_extra_body, + ) + self._client = DuplexClient( + self._url, + model=model, + config=config, + session_id=session_id, + reconnect=None, + heartbeat_interval_s=None, + handshake_timeout_s=timeout_s, + ) + await self._client.__aenter__() + self._consume_task = asyncio.create_task(self.events.consume(self._client)) + + async def send(self, event: dict[str, object]) -> None: + assert self._client is not None + await self._client.send(event) + + async def acknowledge_playback(self) -> None: + from vllm_omni.clients.duplex import acknowledge_collected_playback + + assert self._client is not None + await acknowledge_collected_playback(self._client, self.events) + + async def close_session(self, *, timeout_s: float = 20.0) -> None: + assert self._client is not None + await self._client.close(timeout_s=timeout_s) + + async def async_request_openai_realtime_duplex( request_func_input: RequestFuncInput, session: aiohttp.ClientSession, pbar: tqdm | None = None, ) -> MixRequestFuncOutput: + from vllm_omni.clients.duplex import summarize_session_request_metrics, wait_for_condition + del session if getattr(request_func_input, "omniinteract_case", None) is not None: return await _async_request_omniinteract(request_func_input, pbar=pbar) @@ -1715,7 +1804,7 @@ async def async_request_openai_realtime_duplex( turn_prompts = [("", request_func_input.prompt)] session_id = f"seed-tts-{request_func_input.request_id or uuid.uuid4().hex}" try: - async with RealtimeDuplexClient(_realtime_websocket_url(request_func_input.api_url)) as client: + async with _RealtimeTTSProbe(_realtime_websocket_url(request_func_input.api_url)) as client: await client.configure( request_func_input.model_name or request_func_input.model, output_audio_format="pcm16", @@ -1755,7 +1844,7 @@ async def async_request_openai_realtime_duplex( } ) await client.send({"type": "response.create"}) - await wait_for( + await wait_for_condition( lambda: ( client.events.count("response.done") > done_before or len(client.events.errors()) > errors_before @@ -1794,6 +1883,9 @@ async def async_request_openai_realtime_duplex( "utterance_id": utterance_id or None, "response_id": response_id, **request_metrics, + # The client reports raw data only; the RTF is derived + # here with the canonical server-side definition. + "rtf": _audio_rtf_from_raw_metric(request_metrics), } ) response_audio = client.events.audio_bytes(response_id) diff --git a/vllm_omni/clients/__init__.py b/vllm_omni/clients/__init__.py new file mode 100644 index 000000000..924582bf4 --- /dev/null +++ b/vllm_omni/clients/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Client-side libraries for vLLM-Omni serving APIs. + +Modules under this package speak to a running vLLM-Omni server over the +network. They are for applications and tests; server runtime code +(``vllm_omni.engine``, ``vllm_omni.entrypoints``, ``vllm_omni.model_executor``) +must never import from here. +""" diff --git a/vllm_omni/clients/duplex.py b/vllm_omni/clients/duplex.py new file mode 100644 index 000000000..eb8a6ded0 --- /dev/null +++ b/vllm_omni/clients/duplex.py @@ -0,0 +1,1688 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Public asynchronous client for the vLLM-Omni full-duplex Realtime API. + +Connects to ``/v1/realtime?duplex=1`` (the normative duplex contract) and +provides typed events, response demultiplexing, incremental playback +acking, and transparent session resume on transport drops. + +Example:: + + from vllm_omni.clients.duplex import DuplexClient + from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config + + cfg = create_duplex_session_config(ref_audio=audio_data_url(wav_bytes)) + async with DuplexClient("ws://localhost:8099", model=model, config=cfg) as client: + await client.stream_pcm(pcm16) + await client.commit() + async for response in client.responses(): + if response.decision == "listen": + continue + async for chunk in response.audio(): + play(chunk) + await client.ack_playback(response.played_ms) + break + +Media capture/playback, VAD, and resampling stay in the application; this +module moves bytes and events. +""" + +from __future__ import annotations + +import asyncio +import json +import math +import random +import time +import wave +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from uuid import uuid4 + +import pybase64 as base64 + +__all__ = [ + "DUPLEX_FIRST_UNIT_MS", + "DUPLEX_UNIT_MS", + "PCM16_BYTES_PER_SAMPLE", + "PCM16_SAMPLE_RATE", + "AudioFormat", + "AudioDelta", + "ConnectionResumed", + "DuplexClient", + "DuplexClientError", + "DuplexConnectionError", + "DuplexEvent", + "DuplexProtocolError", + "DuplexSessionClosedError", + "ErrorEvent", + "EventCollector", + "ListenDecision", + "ReconnectPolicy", + "ResponseCreated", + "ResponseDone", + "ResponseHandle", + "SessionConfig", + "SessionCreated", + "SpeakDecision", + "WebSocketTransport", + "TextDelta", + "TranscriptDelta", + "acknowledge_collected_playback", + "audio_data_url", + "build_realtime_url", + "chunk_period_ms", + "duplex_unit_boundary_ms", + "has_residual_model_unit", + "image_data_url", + "read_pcm16_wav", + "reference_audio_data_url", + "summarize_session_request_metrics", + "wait_for_condition", + "write_pcm16_wav", +] + +# The default duplex input wire format (16 kHz mono PCM16), kept as plain +# constants for callers doing byte/duration arithmetic outside AudioFormat. +PCM16_SAMPLE_RATE = 16_000 +PCM16_BYTES_PER_SAMPLE = 2 + +# Server-side model unit boundaries, in cumulative appended audio. +# Stage0 configures the streaming mel processor with first_chunk_ms=1035 and +# chunk_ms=1000; the processor aligns the first chunk down to a hop_length (160 +# samples) multiple, so unit 0 closes at 16480 samples and every later unit +# closes 16000 samples after it. Camera frames must ride the append that closes +# a unit, otherwise Stage0 cannot bind them to that unit's audio. +DUPLEX_FIRST_UNIT_MS = 1030 +DUPLEX_UNIT_MS = 1000 + + +def duplex_unit_boundary_ms(unit_index: int) -> int: + """Cumulative appended audio, in ms, that closes model unit ``unit_index``.""" + return DUPLEX_FIRST_UNIT_MS + max(0, int(unit_index)) * DUPLEX_UNIT_MS + + +# Internal transport/backpressure bounds (promote to constructor parameters +# only if a real consumer needs to tune them). +_MAX_BUFFERED_EVENTS = 1024 +_MAX_FRAME_BYTES = 64 << 20 + + +def _bare_base64(data: str) -> str: + """Strip a ``data:;base64,`` prefix; the wire carries bare base64.""" + if data.startswith("data:"): + _, _, payload = data.partition(";base64,") + if payload: + return payload + return data + + +def _put_drop_oldest(queue: asyncio.Queue, item: object) -> None: + """Deliver ``item`` without blocking, dropping the oldest entry when full. + + Every internal feed path uses this: a consumer that stops draining its + queue loses its oldest buffered events instead of parking the single + reader task, which must keep dispatching (and acking) for the rest of + the session. + """ + try: + queue.put_nowait(item) + except asyncio.QueueFull: + try: + queue.get_nowait() + except asyncio.QueueEmpty: # pragma: no cover - racing consumer + pass + queue.put_nowait(item) + + +# --------------------------------------------------------------------------- +# Errors + + +class DuplexClientError(Exception): + """Base class for all duplex client errors.""" + + +class DuplexConnectionError(DuplexClientError): + """The WebSocket transport could not be established or failed.""" + + +class DuplexProtocolError(DuplexClientError): + """The server rejected a request with an ``error`` event.""" + + def __init__( + self, + message: str, + *, + code: str | None = None, + event_id: str | None = None, + raw: dict[str, object] | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.event_id = event_id + self.raw = dict(raw or {}) + + +class DuplexSessionClosedError(DuplexClientError): + """The session ended and the client cannot continue.""" + + def __init__(self, reason: str) -> None: + super().__init__(f"duplex session closed: {reason}") + self.reason = reason + + +# --------------------------------------------------------------------------- +# Configuration + +_BYTES_PER_SAMPLE = {"pcm16": 2, "pcm_f32le": 4} + + +@dataclass(frozen=True) +class AudioFormat: + """One PCM wire format: encoding name plus sample rate.""" + + encoding: str + sample_rate_hz: int + + @property + def bytes_per_sample(self) -> int: + try: + return _BYTES_PER_SAMPLE[self.encoding] + except KeyError: + raise ValueError(f"Unknown audio encoding: {self.encoding!r}") from None + + def duration_ms(self, byte_count: int) -> float: + return byte_count * 1000.0 / (self.sample_rate_hz * self.bytes_per_sample) + + def byte_count(self, duration_ms: float) -> int: + samples = round(self.sample_rate_hz * duration_ms / 1000.0) + return int(samples) * self.bytes_per_sample + + +@dataclass +class SessionConfig: + """Model-agnostic duplex session configuration. + + Model-specific knobs ride in ``extra_body``; each duplex model ships a + ``create_duplex_session_config`` preset in its client module (e.g. + ``vllm_omni.clients.minicpmo_4_5``), so the client itself stays + model-neutral. + """ + + modalities: tuple[str, ...] = ("audio", "text") + input_audio: AudioFormat = AudioFormat("pcm16", 16_000) + output_audio: AudioFormat = AudioFormat("pcm16", 24_000) + instructions: str | None = None + ref_audio: str | None = None + voice: str | None = None + auto_response: bool = True + temperature: float | None = None + overlap_policy: str | None = None + playback_commit_policy: str | None = None + turn_detection: dict[str, object] | None = None + idle_timeout_s: float | None = None + extra_body: dict[str, object] = field(default_factory=dict) + + def to_session_payload(self, *, model: str, session_id: str | None = None) -> dict[str, object]: + payload: dict[str, object] = { + "model": model, + "modalities": list(self.modalities), + "input_audio_format": self.input_audio.encoding, + "output_audio_format": self.output_audio.encoding, + # The encodings alone do not pin the rates (presets differ from the + # server defaults), so declare both explicitly: the server reads + # the input rate from the top-level field and the output rate from + # the audio.output block. + "sample_rate_hz": self.input_audio.sample_rate_hz, + "audio": { + "input": {"sample_rate_hz": self.input_audio.sample_rate_hz}, + "output": {"sample_rate_hz": self.output_audio.sample_rate_hz}, + }, + "turn_detection": self.turn_detection, + } + if session_id: + payload["session_id"] = session_id + if self.instructions is not None: + payload["instructions"] = self.instructions + if self.ref_audio is not None: + payload["ref_audio"] = self.ref_audio + if self.voice is not None: + payload["voice"] = self.voice + if self.temperature is not None: + payload["temperature"] = self.temperature + if self.overlap_policy is not None: + payload["overlap_policy"] = self.overlap_policy + if self.playback_commit_policy is not None: + payload["playback_commit_policy"] = self.playback_commit_policy + if self.idle_timeout_s is not None: + payload["idle_timeout_s"] = self.idle_timeout_s + extra_body: dict[str, object] = {"auto_response": self.auto_response} + extra_body.update(self.extra_body) + payload["extra_body"] = extra_body + return payload + + +@dataclass(frozen=True) +class ReconnectPolicy: + """Jittered exponential backoff for ``session.resume`` reconnects.""" + + max_attempts: int = 5 + backoff_s: tuple[float, float] = (0.25, 4.0) + + def delay(self, attempt: int) -> float: + low, high = self.backoff_s + ceiling = min(high, low * (2.0 ** max(0, attempt))) + return random.uniform(low, max(low, ceiling)) + + +# --------------------------------------------------------------------------- +# Typed events + + +class DuplexEvent: + """Thin typed wrapper over one server event dict.""" + + __slots__ = ("raw",) + + def __init__(self, raw: dict[str, object]) -> None: + self.raw = raw + + @property + def type(self) -> str: + value = self.raw.get("type") + return value if isinstance(value, str) else "" + + @property + def event_id(self) -> str | None: + value = self.raw.get("event_id") + return value if isinstance(value, str) else None + + @property + def session_id(self) -> str | None: + value = self.raw.get("session_id") + return value if isinstance(value, str) else None + + @property + def response_id(self) -> str | None: + value = self.raw.get("response_id") + if isinstance(value, str): + return value + response = self.raw.get("response") + if isinstance(response, dict): + value = response.get("id") + if isinstance(value, str): + return value + return None + + @property + def item_id(self) -> str | None: + value = self.raw.get("item_id") + return value if isinstance(value, str) else None + + @property + def server_event_seq(self) -> int | None: + value = self.raw.get("server_event_seq") + return value if isinstance(value, int) else None + + @property + def sample_rate_hz(self) -> int | None: + value = self.raw.get("sample_rate_hz") + return value if isinstance(value, int) and value > 0 else None + + @property + def audio(self) -> bytes | None: + delta = self.raw.get("delta") or self.raw.get("audio") + if not isinstance(delta, str) or not delta: + return None + try: + return base64.b64decode(delta) + except ValueError: + return None + + @property + def text(self) -> str | None: + for key in ("text", "delta", "transcript"): + value = self.raw.get(key) + if isinstance(value, str): + return value + return None + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"{type(self).__name__}(type={self.type!r})" + + +class SessionCreated(DuplexEvent): + @property + def session(self) -> dict[str, object]: + value = self.raw.get("session") + return value if isinstance(value, dict) else {} + + @property + def resume_token(self) -> str | None: + value = self.raw.get("resume_token") + return value if isinstance(value, str) else None + + @property + def incarnation(self) -> int: + value = self.raw.get("incarnation") + return value if isinstance(value, int) else 0 + + +class SessionResumed(SessionCreated): + pass + + +class SessionClosed(DuplexEvent): + pass + + +class SessionExpired(DuplexEvent): + pass + + +class ResponseCreated(DuplexEvent): + pass + + +class ResponseDone(DuplexEvent): + pass + + +class AudioDelta(DuplexEvent): + pass + + +class TranscriptDelta(DuplexEvent): + pass + + +class TextDelta(DuplexEvent): + pass + + +class ListenDecision(DuplexEvent): + pass + + +class SpeakDecision(DuplexEvent): + pass + + +class ConnectionResumed(DuplexEvent): + """Synthetic client event: the transport dropped and was resumed.""" + + +class ErrorEvent(DuplexEvent): + @property + def _error(self) -> dict[str, object] | None: + value = self.raw.get("error") + return value if isinstance(value, dict) else None + + @property + def code(self) -> str | None: + error = self._error + value = (error or self.raw).get("code") + return value if isinstance(value, str) else None + + @property + def message(self) -> str: + error = self._error + if error is not None: + value = error.get("message") + if isinstance(value, str): + return value + value = self.raw.get("error") + return value if isinstance(value, str) else json.dumps(self.raw) + + @property + def related_event_id(self) -> str | None: + error = self._error + value = error.get("event_id") if error is not None else None + return value if isinstance(value, str) else None + + def to_exception(self) -> DuplexProtocolError: + return DuplexProtocolError( + self.message, + code=self.code, + event_id=self.related_event_id, + raw=self.raw, + ) + + +_EVENT_TYPES: dict[str, type[DuplexEvent]] = { + "session.created": SessionCreated, + "session.resumed": SessionResumed, + "session.closed": SessionClosed, + "session.expired": SessionExpired, + "response.created": ResponseCreated, + "response.done": ResponseDone, + "response.audio.delta": AudioDelta, + "response.output_audio.delta": AudioDelta, + "response.audio_transcript.delta": TranscriptDelta, + "response.output_audio_transcript.delta": TranscriptDelta, + "response.text.delta": TextDelta, + "response.output_text.delta": TextDelta, + "response.listen": ListenDecision, + "response.speak": SpeakDecision, + "connection.resumed": ConnectionResumed, + "error": ErrorEvent, +} + +_AUDIO_DELTA_TYPES = frozenset(name for name, event_cls in _EVENT_TYPES.items() if event_cls is AudioDelta) + +_FATAL_RESUME_CODES = frozenset( + { + "invalid_session_resume", + "session_resume_expired", + "unsupported_session_resume", + "invalid_resume_token", + "session_resume_conflict", + } +) + + +def wrap_event(raw: dict[str, object]) -> DuplexEvent: + event_type = raw.get("type") + event_cls = _EVENT_TYPES.get(event_type) if isinstance(event_type, str) else None + return (event_cls or DuplexEvent)(raw) + + +@dataclass(frozen=True) +class _ClosedMarker: + reason: str + expected: bool + + +@dataclass(frozen=True) +class _ErrorMarker: + error: ErrorEvent + + +# --------------------------------------------------------------------------- +# Response demultiplexing + + +class ResponseHandle: + """One ``response.created`` → terminal lifecycle, demultiplexed. + + A handle reaches :meth:`DuplexClient.responses` only once its decision is + known: when the response starts producing output (``decision`` is + ``"speak"`` or about to become it) or when a terminal listen finishes it + (``decision == "listen"``, no audio). Standalone listen decisions surface + as already-finished handles too, so a turn loop stays uniform. + """ + + def __init__( + self, + *, + response_id: str | None, + output_format: AudioFormat, + max_buffered_events: int, + decision: str | None = None, + created_event: ResponseCreated | None = None, + ) -> None: + self.response_id = response_id + self.decision = decision + self.transcript = "" + self.text = "" + self.played_ms = 0.0 + self.created_event = created_event + self.done_event: DuplexEvent | None = None + self._output_format = output_format + self._audio_queue: asyncio.Queue[tuple[bytes, int | None] | None] = asyncio.Queue(maxsize=max_buffered_events) + self._done = asyncio.Event() + self._announced = False + + @property + def finished(self) -> bool: + return self._done.is_set() + + async def wait(self, timeout_s: float | None = None) -> None: + if timeout_s is None: + await self._done.wait() + else: + await asyncio.wait_for(self._done.wait(), timeout_s) + + async def audio(self) -> AsyncIterator[bytes]: + """Yield decoded output-audio chunks until the response finishes. + + ``played_ms`` advances as chunks are yielded, so it reflects what the + application has taken for playback — feed it to + :meth:`DuplexClient.ack_playback`. A handle whose audio is never + drained drops its oldest buffered chunks once the buffer fills; it + never stalls the client's reader. + """ + while True: + item = await self._audio_queue.get() + if item is None: + return + chunk, sample_rate_hz = item + rate = sample_rate_hz or self._output_format.sample_rate_hz + self.played_ms += len(chunk) * 1000.0 / (rate * self._output_format.bytes_per_sample) + yield chunk + + def _feed(self, event: DuplexEvent) -> None: + if self._done.is_set(): + return + if isinstance(event, AudioDelta): + chunk = event.audio + if chunk: + _put_drop_oldest(self._audio_queue, (chunk, event.sample_rate_hz)) + elif isinstance(event, TranscriptDelta): + self.transcript += event.text or "" + elif isinstance(event, TextDelta): + self.text += event.text or "" + elif isinstance(event, SpeakDecision): + self.decision = "speak" + elif isinstance(event, ListenDecision): + # A terminal listen on a response that already spoke is the + # model yielding the turn; it must not downgrade the decision. + if self.decision is None: + self.decision = "listen" + + def _finish(self, event: DuplexEvent | None) -> None: + if self._done.is_set(): + return + self.done_event = event + self._done.set() + # A full queue cannot block _finish: the sentinel is delivered by a + # non-blocking put with a drop-oldest fallback. + _put_drop_oldest(self._audio_queue, None) + + +# --------------------------------------------------------------------------- +# Client + + +class WebSocketTransport(Protocol): + """Minimal WebSocket surface the client needs (satisfied by ``websockets``).""" + + async def send(self, data: str) -> None: ... + + async def recv(self) -> str | bytes: ... + + async def close(self) -> None: ... + + +ConnectFn = Callable[[str], Awaitable["WebSocketTransport"]] + + +class DuplexClient: + """Async client for one duplex session over ``/v1/realtime?duplex=1``. + + ``session_id`` only names the session this client creates: entering the + client always performs the ``session.update`` handshake. Resuming an + existing session is supported only as automatic reconnect within the same + client instance (see ``ReconnectPolicy``); taking over a session from a + new client requires the wire-level ``session.resume`` handshake + (``resume_token``, ``incarnation``, ``last_received_server_event_seq``), + which this client does not expose. + """ + + def __init__( + self, + url: str, + *, + model: str, + config: SessionConfig | None = None, + session_id: str | None = None, + reconnect: ReconnectPolicy | None = ReconnectPolicy(), + heartbeat_interval_s: float | None = 30.0, + handshake_timeout_s: float = 30.0, + connect: ConnectFn | None = None, + ) -> None: + self.url = url + self.model = model + self.config = config or SessionConfig() + self.session_id = session_id + self.session_info: dict[str, object] = {} + self.incarnation = 0 + self.resume_token: str | None = None + self._reconnect = reconnect + self._heartbeat_interval_s = heartbeat_interval_s + self._handshake_timeout_s = handshake_timeout_s + self._connect_fn: ConnectFn = connect or self._default_connect + self._ws: WebSocketTransport | None = None + self._reader_task: asyncio.Task[None] | None = None + self._heartbeat_task: asyncio.Task[None] | None = None + self._subscribers: list[asyncio.Queue[DuplexEvent | _ClosedMarker]] = [] + self._response_queue: asyncio.Queue[ResponseHandle | _ClosedMarker | _ErrorMarker] = asyncio.Queue( + maxsize=_MAX_BUFFERED_EVENTS + ) + self._responses: dict[str, ResponseHandle] = {} + self._last_server_event_seq: int | None = None + self._input_audio_end_ms = 0.0 + self._closing = False + self._closed = asyncio.Event() + self._closed_marker: _ClosedMarker | None = None + + # -- lifecycle ---------------------------------------------------------- + + async def __aenter__(self) -> DuplexClient: + self._ws = await self._connect_fn(self._target_url()) + handshake_queue = self._add_subscriber() + try: + self._reader_task = asyncio.create_task(self._read_loop(), name="duplex-client-reader") + await self.send( + { + "type": "session.update", + "session": self.config.to_session_payload(model=self.model, session_id=self.session_id), + } + ) + created = await self._wait_on_queue( + handshake_queue, + ("session.created",), + timeout_s=self._handshake_timeout_s, + ) + except BaseException: + await self._teardown() + raise + finally: + self._remove_subscriber(handshake_queue) + assert isinstance(created, SessionCreated) + self._adopt_session(created) + if self._heartbeat_interval_s is not None: + self._heartbeat_task = asyncio.create_task(self._heartbeat_loop(), name="duplex-client-heartbeat") + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + try: + if exc_type is None and not self._closed.is_set(): + await self.close() + elif not self._closed.is_set(): + # An application error (or cancellation) is propagating. Still + # tell the server the session is over: dropping the transport + # silently reads as a resumable disconnect and holds the + # session slot for the full disconnect grace period. Best + # effort only — the teardown below closes the transport + # whether or not the close reaches the server. + self._closing = True + try: + await self.send({"type": "session.close"}) + except DuplexClientError: + pass + finally: + await self._teardown() + + async def close(self, *, timeout_s: float = 20.0) -> None: + """Send ``session.close`` and wait for the server to confirm.""" + if self._closed.is_set(): + return + self._closing = True + try: + await self.send({"type": "session.close"}) + except (DuplexConnectionError, DuplexSessionClosedError): + self._finalize("closed", expected=True) + return + try: + await asyncio.wait_for(self._closed.wait(), timeout_s) + except asyncio.TimeoutError: + self._finalize("close timed out", expected=True) + + # -- input ---------------------------------------------------------------- + + async def append_audio( + self, + pcm: bytes, + *, + is_speech: bool | None = None, + video_frames: Sequence[str] | None = None, + ) -> None: + """Append one chunk of input audio in the session's input format. + + A turn is ended with :meth:`commit`, never by a flag on the append. + + ``video_frames`` takes base64 JPEG/PNG strings (one per ~1 s unit for + omni models). Data URLs (:func:`image_data_url`) are accepted too; + their prefix is stripped, since the wire contract carries bare base64. + """ + input_format = self.config.input_audio + duration_ms = input_format.duration_ms(len(pcm)) + self._input_audio_end_ms += duration_ms + event: dict[str, object] = { + "type": "input_audio_buffer.append", + "audio": base64.b64encode(pcm).decode("ascii"), + "format": input_format.encoding, + "sample_rate_hz": input_format.sample_rate_hz, + "duration_ms": round(duration_ms), + "audio_end_ms": round(self._input_audio_end_ms), + } + if is_speech is not None: + event["is_speech"] = is_speech + if video_frames: + event["video_frames"] = [_bare_base64(frame) for frame in video_frames] + await self.send(event) + + async def stream_pcm( + self, + pcm: bytes, + *, + chunk_ms: int = 200, + realtime: bool = True, + is_speech: bool | None = None, + video_frames: Sequence[str] | None = None, + stacked_video_frames: Sequence[str | None] | None = None, + ) -> int: + """Slice ``pcm`` into ``chunk_ms`` chunks, pace, append; interleave frames. + + ``video_frames`` holds base64/data-URL JPEG frames in capture order, + one per second of the clip. Frame ``k`` rides the append that closes + model unit ``k`` (see :func:`duplex_unit_boundary_ms`), which + reproduces the official ``streaming_prefill(audio_waveform=<1 s>, + frame_list=[frame])`` pairing: a second of audio and the picture + captured during it enter the same unit. Sending on whole-second + boundaries instead would strand frame 0 on an append that cannot + close a unit yet, and shift every later frame one unit ahead of its + audio. + + ``stacked_video_frames`` is the optional parallel track of composites + (see ``vllm_omni.experimental.fullduplex.video_stacking``): entry ``k`` + tiles the sub-frames captured *inside* unit ``k`` and rides the same + append right after the base frame. ``None`` entries send the base + frame alone. + + Returns the number of base frames actually sent (a clip shorter than + the frame list leaves the tail unsent). + """ + input_format = self.config.input_audio + chunk_bytes = max(input_format.byte_count(chunk_ms), input_format.bytes_per_sample) + frames = list(video_frames or []) + stacked = list(stacked_video_frames or []) + + def units() -> Iterator[tuple[bytes, list[str] | None]]: + audio_end_ms = 0.0 + frames_sent = 0 + for offset in range(0, len(pcm), chunk_bytes): + chunk = pcm[offset : offset + chunk_bytes] + audio_end_ms += input_format.duration_ms(len(chunk)) + if not frames or audio_end_ms < duplex_unit_boundary_ms(frames_sent): + yield chunk, None + continue + # A video advances one frame per unit and holds its last frame + # when the audio outlives the clip; a still image is a + # one-element list and therefore repeats. + index = min(frames_sent, len(frames) - 1) + composite = stacked[index] if index < len(stacked) else None + frames_sent += 1 + yield chunk, [frames[index]] if composite is None else [frames[index], composite] + + frames_sent = 0 + for chunk, unit_frames in units(): + if not chunk: + continue + await self.append_audio(chunk, is_speech=is_speech, video_frames=unit_frames) + if unit_frames: + frames_sent += 1 + if realtime: + await asyncio.sleep(input_format.duration_ms(len(chunk)) / 1000.0) + return frames_sent + + async def commit(self, *, final: bool = True, create_response: bool | None = None) -> None: + event: dict[str, object] = {"type": "input_audio_buffer.commit", "final": final} + if create_response is not None: + event["response_create"] = create_response + await self.send(event) + + # -- interruption --------------------------------------------------------- + + async def cancel_response(self, response_id: str | None = None) -> None: + """Cancel the active (or a specific) response. + + Together with :meth:`clear_input` this composes a client-forced + barge-in for sessions whose capabilities advertise + ``supports_barge_in``; the currently bundled duplex models do not, + so their overlap handling is model-owned. + """ + event: dict[str, object] = {"type": "response.cancel"} + if response_id is not None: + event["response_id"] = response_id + await self.send(event) + + async def clear_input(self) -> None: + await self.send({"type": "input_audio_buffer.clear"}) + + # -- playback contract ------------------------------------------------------ + + async def ack_playback( + self, + played_ms: float, + *, + response_id: str | None = None, + item_id: str | None = None, + committed_ms: float | None = None, + ) -> None: + """Report cumulative playback progress; call periodically while playing.""" + event: dict[str, object] = { + "type": "playback.ack", + "played_ms": int(played_ms), + "committed_ms": int(committed_ms if committed_ms is not None else played_ms), + } + if response_id is not None: + event["response_id"] = response_id + if item_id is not None: + event["item_id"] = item_id + await self.send(event) + + # -- output ---------------------------------------------------------------- + + def __aiter__(self) -> AsyncIterator[DuplexEvent]: + return self.events() + + async def events(self) -> AsyncIterator[DuplexEvent]: + """Iterate over every server event from now on (typed).""" + queue = self._add_subscriber() + try: + while True: + item = await queue.get() + if isinstance(item, _ClosedMarker): + if item.expected: + return + raise DuplexSessionClosedError(item.reason) + yield item + finally: + self._remove_subscriber(queue) + + async def responses(self) -> AsyncIterator[ResponseHandle]: + """Iterate over response lifecycles (single consumer). + + Raises :class:`DuplexProtocolError` when the server rejects a request + with an ``error`` event while waiting — a rejected send (an invalid + video frame, an empty commit) produces no response, so without this + the wait would never end. Re-enter the iterator to keep consuming + after handling the error. + """ + while True: + item = await self._response_queue.get() + if isinstance(item, _ClosedMarker): + if item.expected: + return + raise DuplexSessionClosedError(item.reason) + if isinstance(item, _ErrorMarker): + raise item.error.to_exception() + yield item + + async def wait_for(self, *types: str, timeout_s: float) -> DuplexEvent: + """Wait for the next event whose type is in ``types``. + + Raises :class:`DuplexProtocolError` if an ``error`` event arrives + first, and :class:`DuplexSessionClosedError` if the session ends first. + """ + queue = self._add_subscriber() + try: + return await self._wait_on_queue(queue, types, timeout_s=timeout_s) + finally: + self._remove_subscriber(queue) + + async def send(self, event: dict[str, object]) -> str: + """Send one raw client event; returns the (possibly stamped) event_id.""" + if self._closed.is_set(): + marker = self._closed_marker + raise DuplexSessionClosedError(marker.reason if marker else "closed") + payload = dict(event) + payload.setdefault("event_id", f"evt_{uuid4().hex}") + try: + await self._ws.send(json.dumps(payload)) + except asyncio.CancelledError: + raise + except Exception as exc: + raise DuplexConnectionError(f"send failed: {exc}") from exc + return payload["event_id"] + + # -- internals --------------------------------------------------------------- + + def _target_url(self) -> str: + parts = urlsplit(self.url) + path = parts.path if parts.path not in ("", "/") else "/v1/realtime" + query = dict(parse_qsl(parts.query, keep_blank_values=True)) + query.setdefault("duplex", "1") + query.setdefault("model", self.model) + # The client always opens with an explicit session.update (and resumes + # with an explicit session.resume). Autostart would race that handshake: + # with a model query param, the server creates a bare default session + # before reading the first client event, silently dropping ref_audio + # and the model-specific extra_body. Force it off even when the caller's + # URL carries autostart=1 — this client cannot operate over it. + query["autostart"] = "0" + return urlunsplit((parts.scheme or "ws", parts.netloc, path, urlencode(query), parts.fragment)) + + async def _default_connect(self, url: str) -> WebSocketTransport: + try: + import websockets + except ImportError as exc: # pragma: no cover - websockets is a pinned dep + raise DuplexConnectionError("The duplex client requires the 'websockets' package") from exc + try: + return await websockets.connect(url, max_size=_MAX_FRAME_BYTES) + except asyncio.CancelledError: + raise + except Exception as exc: + raise DuplexConnectionError(f"connect to {url} failed: {exc}") from exc + + def _adopt_session(self, event: SessionCreated) -> None: + # A resume activation carries no nested session object; keep the + # session_info captured at the original handshake in that case. + if event.session: + self.session_info = event.session + session_id = event.session.get("session_id") or event.session.get("id") or event.session_id + if isinstance(session_id, str) and session_id: + self.session_id = session_id + self.incarnation = event.incarnation + if event.resume_token: + self.resume_token = event.resume_token + + def _announce(self, handle: ResponseHandle) -> None: + """Deliver a handle to responses() exactly once, decision known.""" + if not handle._announced: + handle._announced = True + _put_drop_oldest(self._response_queue, handle) + + def _add_subscriber(self) -> asyncio.Queue[DuplexEvent | _ClosedMarker]: + queue: asyncio.Queue[DuplexEvent | _ClosedMarker] = asyncio.Queue(maxsize=_MAX_BUFFERED_EVENTS) + if self._closed.is_set() and self._closed_marker is not None: + queue.put_nowait(self._closed_marker) + else: + self._subscribers.append(queue) + return queue + + def _remove_subscriber(self, queue: asyncio.Queue[DuplexEvent | _ClosedMarker]) -> None: + try: + self._subscribers.remove(queue) + except ValueError: + pass + + async def _wait_on_queue( + self, + queue: asyncio.Queue[DuplexEvent | _ClosedMarker], + types: tuple[str, ...], + *, + timeout_s: float, + ) -> DuplexEvent: + deadline = time.monotonic() + timeout_s + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise asyncio.TimeoutError(f"timed out waiting for {types}") + item = await asyncio.wait_for(queue.get(), remaining) + if isinstance(item, _ClosedMarker): + raise DuplexSessionClosedError(item.reason) + if isinstance(item, ErrorEvent) and item.type not in types: + raise item.to_exception() + if item.type in types: + return item + + async def _read_loop(self) -> None: + while True: + try: + while True: + raw = await self._ws.recv() + if isinstance(raw, (bytes, bytearray)): + continue + try: + data = json.loads(raw) + except json.JSONDecodeError: + continue + if isinstance(data, dict): + await self._dispatch(data) + except asyncio.CancelledError: + raise + except Exception as exc: + if self._closing or self._closed.is_set(): + self._finalize("closed", expected=True) + return + if await self._attempt_resume(): + continue + self._finalize(f"connection lost: {exc}", expected=False) + return + + async def _dispatch(self, data: dict[str, object]) -> None: + seq = data.get("server_event_seq") + if isinstance(seq, int): + if self._last_server_event_seq is not None and seq <= self._last_server_event_seq: + return # duplicate delivered by resume replay + self._last_server_event_seq = seq + event = wrap_event(data) + + if isinstance(event, SessionResumed): + self._adopt_session(event) + elif isinstance(event, SessionCreated): + self._adopt_session(event) + elif isinstance(event, ResponseCreated): + response_id = event.response_id + if response_id and response_id not in self._responses: + handle = ResponseHandle( + response_id=response_id, + output_format=self.config.output_audio, + max_buffered_events=_MAX_BUFFERED_EVENTS, + created_event=event, + ) + self._responses[response_id] = handle + # Not yielded yet: the handle reaches responses() once its + # decision is known — at the first output below, or at its + # terminal event — so a turn loop can branch on `decision` + # the moment it receives the handle. + elif isinstance(event, (AudioDelta, TranscriptDelta, TextDelta, SpeakDecision)): + handle = self._responses.get(event.response_id or "") + if handle is not None: + handle._feed(event) + self._announce(handle) + elif isinstance(event, ListenDecision): + # Only a listen carrying a response_id is that response's + # terminal decision (the server stamps it exactly there); an + # id-less listen is a standalone decision beat and must never + # close an in-flight response. + handle = self._responses.pop(event.response_id or "", None) + if handle is not None: + handle._feed(event) + handle._finish(event) + self._announce(handle) + else: + listen_handle = ResponseHandle( + response_id=event.response_id, + output_format=self.config.output_audio, + max_buffered_events=_MAX_BUFFERED_EVENTS, + decision="listen", + ) + listen_handle._announced = True + listen_handle._finish(event) + _put_drop_oldest(self._response_queue, listen_handle) + elif isinstance(event, ResponseDone): + handle = self._responses.pop(event.response_id or "", None) + if handle is not None: + if handle.decision is None: + handle.decision = "speak" + handle._finish(event) + self._announce(handle) + elif isinstance(event, ErrorEvent): + # A rejected request produces no response; surface the rejection + # on the response path too so a consumer waiting in responses() + # is not left waiting forever (see the responses() docstring). + _put_drop_oldest(self._response_queue, _ErrorMarker(event)) + + for queue in list(self._subscribers): + _put_drop_oldest(queue, event) + + if isinstance(event, SessionClosed): + self._finalize("closed", expected=True) + elif isinstance(event, SessionExpired): + reason = data.get("reason") + self._finalize(f"expired: {reason}" if reason else "expired", expected=False) + elif event.type == "session.resync_required": + # The server stopped journaling this session's events, so any + # retained resume credential will be refused on the next + # session.resume. Drop it now: a later transport failure then + # finalizes immediately instead of burning reconnect attempts. + self.resume_token = None + + if isinstance(seq, int) and not self._closed.is_set(): + try: + await self._ws.send(json.dumps({"type": "session.event_ack", "server_event_seq": seq})) + except asyncio.CancelledError: + raise + except Exception: + pass # the reader will observe the transport failure itself + + async def _attempt_resume(self) -> bool: + policy = self._reconnect + if policy is None or not self.resume_token or not self.session_id: + return False + for attempt in range(policy.max_attempts): + await asyncio.sleep(policy.delay(attempt)) + try: + ws = await self._connect_fn(self._target_url()) + except asyncio.CancelledError: + raise + except Exception: + continue + try: + await ws.send( + json.dumps( + { + "type": "session.resume", + "session_id": self.session_id, + "incarnation": self.incarnation, + "resume_token": self.resume_token, + "last_received_server_event_seq": self._last_server_event_seq or 0, + } + ) + ) + deadline = time.monotonic() + self._handshake_timeout_s + pending: list[dict[str, object]] = [] + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise asyncio.TimeoutError + raw = await asyncio.wait_for(ws.recv(), remaining) + if isinstance(raw, (bytes, bytearray)): + continue + data = json.loads(raw) + if not isinstance(data, dict): + continue + event_type = data.get("type") + if event_type == "session.resumed": + self._ws = ws + await self._dispatch({"type": "connection.resumed", "attempt": attempt}) + await self._dispatch(data) + for entry in pending: + await self._dispatch(entry) + return True + if event_type == "session.resync_required": + await self._close_quietly(ws) + return False + if event_type == "error": + code = ErrorEvent(data).code + if code in _FATAL_RESUME_CODES: + await self._close_quietly(ws) + return False + await self._close_quietly(ws) + break # transient error: next attempt + pending.append(data) + except asyncio.CancelledError: + # ws is not yet self._ws, so _teardown will not close it. + await self._close_quietly(ws) + raise + except Exception: + await self._close_quietly(ws) + continue + return False + + @staticmethod + async def _close_quietly(ws: WebSocketTransport) -> None: + try: + await ws.close() + except Exception: # noqa: BLE001 + pass + + async def _heartbeat_loop(self) -> None: + interval = self._heartbeat_interval_s + assert interval is not None + while not self._closed.is_set(): + await asyncio.sleep(interval) + try: + await self.send({"type": "session.heartbeat"}) + except DuplexConnectionError: + # Transient transport failure — a tick can land inside the + # resume window while the old socket is already dead. Keep + # ticking; the resumed session still needs heartbeats to + # reset the server's receive timeout. If the session is + # really over, _finalize sets _closed and the loop exits. + continue + except DuplexClientError: + return + + def _finalize(self, reason: str, *, expected: bool) -> None: + if self._closed.is_set(): + return + marker = _ClosedMarker(reason=reason, expected=expected) + self._closed_marker = marker + self._closed.set() + for handle in self._responses.values(): + handle._finish(None) + self._responses.clear() + for queue in (*self._subscribers, self._response_queue): + _put_drop_oldest(queue, marker) + + async def _teardown(self) -> None: + self._closing = True + if not self._closed.is_set(): + self._finalize("closed", expected=True) + for task in (self._heartbeat_task, self._reader_task): + if task is not None and not task.done(): + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + if self._ws is not None: + try: + await self._ws.close() + except Exception: # noqa: BLE001 + pass + + +# --------------------------------------------------------------------------- +# Test/benchmark collector + + +def _rounded_ms(value: float) -> float: + return round(float(value), 3) + + +def _interval_summary(values: list[float]) -> dict[str, float | int]: + clean = sorted(_rounded_ms(value) for value in values if math.isfinite(value) and value >= 0) + if not clean: + return {"count": 0, "mean": 0.0, "p50": 0.0, "p95": 0.0, "max": 0.0} + + def nearest_rank(percentile: float) -> float: + index = max(0, math.ceil(percentile * len(clean)) - 1) + return clean[min(index, len(clean) - 1)] + + return { + "count": len(clean), + "mean": _rounded_ms(sum(clean) / len(clean)), + "p50": nearest_rank(0.50), + "p95": nearest_rank(0.95), + "max": clean[-1], + } + + +def _event_stage_metrics(event: dict[str, object]) -> dict[str, object] | None: + candidates: list[object] = [event.get("vllm_omni")] + metadata = event.get("metadata") + if isinstance(metadata, dict): + candidates.extend((metadata, metadata.get("vllm_omni"))) + response = event.get("response") + if isinstance(response, dict): + response_metadata = response.get("metadata") + if isinstance(response_metadata, dict): + candidates.extend((response_metadata, response_metadata.get("vllm_omni"))) + for candidate in candidates: + if not isinstance(candidate, dict): + continue + stage_metrics = candidate.get("stage_metrics") + if isinstance(stage_metrics, dict): + return stage_metrics + return None + + +class EventCollector: + """Accumulate events for assertions and latency summaries (tests/benchmarks). + + Unbounded by design — attach one to a short-lived probe session, not to a + production client that runs for hours. + """ + + def __init__(self) -> None: + self.events: list[dict[str, object]] = [] + self.event_received_at_s: list[float] = [] + self.response_audio: dict[str, list[bytes]] = {} + self.response_ids: list[str] = [] + self.output_sample_rate_hz = 24_000 + + async def consume(self, client: DuplexClient) -> None: + """Subscribe to ``client`` and collect until the session ends.""" + try: + async for event in client.events(): + self.add(event.raw) + except DuplexSessionClosedError: + pass + + @staticmethod + def response_id(event: dict[str, object]) -> str | None: + """Response identity of one raw event (``response_id`` or ``response.id``).""" + return wrap_event(event).response_id + + def add(self, event: dict[str, object] | DuplexEvent, *, received_at_s: float | None = None) -> None: + raw = event.raw if isinstance(event, DuplexEvent) else event + received_at = time.monotonic() if received_at_s is None else float(received_at_s) + stored_event = dict(raw) + stored_event.setdefault("_client_received_at_s", received_at) + self.events.append(stored_event) + self.event_received_at_s.append(received_at) + wrapped = wrap_event(stored_event) + response_id = wrapped.response_id + if stored_event.get("type") == "response.created" and response_id and response_id not in self.response_ids: + self.response_ids.append(response_id) + if stored_event.get("type") in _AUDIO_DELTA_TYPES: + chunk = wrapped.audio + if chunk and response_id: + self.response_audio.setdefault(response_id, []).append(chunk) + sample_rate_hz = wrapped.sample_rate_hz + if sample_rate_hz: + self.output_sample_rate_hz = sample_rate_hz + + def count(self, event_type: str) -> int: + return sum(event.get("type") == event_type for event in self.events) + + def audio_bytes(self, response_id: str | None = None) -> bytes: + if response_id is not None: + return b"".join(self.response_audio.get(response_id, ())) + return b"".join( + chunk for response_id in self.response_ids for chunk in self.response_audio.get(response_id, ()) + ) + + def errors(self) -> list[dict[str, object]]: + return [event for event in self.events if event.get("type") == "error"] + + def response_is_done(self, response_id: str) -> bool: + return any( + event.get("type") == "response.done" and self.response_id(event) == response_id for event in self.events + ) + + def response_text(self, response_id: str) -> str: + """Join all text/transcript deltas for one response identity.""" + return "".join( + str(event.get("delta") or "") + for event in self.events + if self.response_id(event) == response_id + and event.get("type") + in { + "response.audio_transcript.delta", + "response.output_text.delta", + "response.text.delta", + } + ) + + def first_received_at(self, *event_types: str, after_s: float = 0.0) -> float | None: + for event, received_at_s in zip(self.events, self.event_received_at_s, strict=True): + if received_at_s >= after_s and event.get("type") in event_types: + return received_at_s + return None + + def last_received_at(self, event_type: str) -> float | None: + for event, received_at_s in zip( + reversed(self.events), + reversed(self.event_received_at_s), + strict=True, + ): + if event.get("type") == event_type: + return received_at_s + return None + + def timing_summary( + self, + *, + after_s: float, + input_committed_at_s: float | None = None, + response_id: str | None = None, + measurement_origin: dict[str, str] | None = None, + ) -> dict[str, object]: + """Summarize engine token metrics and client-observed audio cadence.""" + stage0_metrics: dict[str, object] | None = None + response_created_at_s: float | None = None + first_text_received_at_s: float | None = None + audio_received_at_s: list[float] = [] + cumulative_audio_ms: list[float] = [] + for event, received_at_s in zip(self.events, self.event_received_at_s, strict=True): + if received_at_s < after_s: + continue + event_response_id = wrap_event(event).response_id + if response_id is not None and event_response_id != response_id: + continue + if event.get("type") == "response.created" and response_created_at_s is None: + response_created_at_s = received_at_s + if ( + event.get("type") + in { + "response.audio_transcript.delta", + "response.output_text.delta", + "response.text.delta", + } + and isinstance(event.get("delta"), str) + and bool(event["delta"]) + and first_text_received_at_s is None + ): + first_text_received_at_s = received_at_s + + stage_metrics = _event_stage_metrics(event) + stage0 = stage_metrics.get("0") if isinstance(stage_metrics, dict) else None + if isinstance(stage0, dict): + stage0_metrics = stage0 + + if event.get("type") not in _AUDIO_DELTA_TYPES: + continue + delta = event.get("delta") or event.get("audio") + if not isinstance(delta, str) or not delta: + continue + audio_received_at_s.append(received_at_s) + metadata = event.get("metadata") + duration_ms = metadata.get("audio_duration_ms") if isinstance(metadata, dict) else None + if isinstance(duration_ms, int | float) and math.isfinite(float(duration_ms)): + cumulative_audio_ms.append(max(0.0, float(duration_ms))) + + result: dict[str, object] = {} + if stage0_metrics is not None: + raw_itls = stage0_metrics.get("vllm_itls_ms") + itls = ( + [float(value) for value in raw_itls if isinstance(value, int | float)] + if isinstance(raw_itls, list) + else [] + ) + result["stage0_tokens"] = { + "source": "engine_stage_metrics", + "output_token_count": int(stage0_metrics.get("num_tokens_out") or 0), + "ttft_ms": float(stage0_metrics.get("vllm_ttft_ms") or 0.0), + "tpot_ms": float(stage0_metrics.get("vllm_tpot_ms") or 0.0), + "itls_ms": itls, + "inter_token_interval_ms": _interval_summary(itls), + } + + if audio_received_at_s: + intervals_ms = [ + (current - previous) * 1000.0 for previous, current in zip(audio_received_at_s, audio_received_at_s[1:]) + ] + chunk_durations_ms: list[float] = [] + previous_duration_ms = 0.0 + for duration_ms in cumulative_audio_ms: + chunk_durations_ms.append( + duration_ms - previous_duration_ms if duration_ms >= previous_duration_ms else duration_ms + ) + previous_duration_ms = duration_ms + interval_summary = _interval_summary(intervals_ms) + result["audio_output"] = { + "source": "client_monotonic_receive", + "chunk_count": len(audio_received_at_s), + "response_created_to_first_audio_ms": ( + _rounded_ms((audio_received_at_s[0] - response_created_at_s) * 1000.0) + if response_created_at_s is not None + else None + ), + "commit_to_first_audio_ms": ( + _rounded_ms((audio_received_at_s[0] - input_committed_at_s) * 1000.0) + if input_committed_at_s is not None + else None + ), + "inter_chunk_interval_ms": interval_summary, + "chunk_duration_ms": _interval_summary(chunk_durations_ms), + "max_chunk_gap_ms": interval_summary["max"], + } + request_started_at_s = input_committed_at_s if input_committed_at_s is not None else response_created_at_s + if request_started_at_s is not None: + audio_duration_ms = ( + max(cumulative_audio_ms) + if cumulative_audio_ms + else self._format_for_bytes().duration_ms(len(self.audio_bytes(response_id))) + ) + audio_generation_ms = max( + 0.0, + (audio_received_at_s[-1] - request_started_at_s) * 1000.0, + ) + # Raw measurements only: derived metrics such as the RTF + # (audio_generation_ms / audio_duration_ms) are computed by + # the caller — e.g. with vllm_omni.metrics.definitions. + # compute_audio_rtf — so metric definitions stay out of this + # dependency-free client module. + result["request_metrics"] = { + "source": "client_monotonic_receive", + "measurement_origin": measurement_origin + or { + "ttft": "input_audio_buffer.commit client send to first non-empty text delta", + "ttfp": "input_audio_buffer.commit client send to first audio packet", + }, + "ttft_ms": ( + _rounded_ms((first_text_received_at_s - request_started_at_s) * 1000.0) + if first_text_received_at_s is not None + else None + ), + "ttfp_ms": _rounded_ms((audio_received_at_s[0] - request_started_at_s) * 1000.0), + "audio_generation_ms": _rounded_ms(audio_generation_ms), + "audio_duration_ms": _rounded_ms(audio_duration_ms), + } + return result + + def _format_for_bytes(self) -> AudioFormat: + return AudioFormat("pcm16", self.output_sample_rate_hz) + + +# --------------------------------------------------------------------------- +# Media helpers + + +def audio_data_url(data: bytes | Path, *, mime: str = "audio/wav") -> str: + """Encode audio bytes (or a file path) as a data URL for ``ref_audio``.""" + payload = data.read_bytes() if isinstance(data, Path) else data + return f"data:{mime};base64," + base64.b64encode(payload).decode("ascii") + + +def image_data_url(data: bytes | Path, *, mime: str = "image/jpeg") -> str: + """Encode an image (or a file path) as a data URL for ``video_frames``.""" + payload = data.read_bytes() if isinstance(data, Path) else data + return f"data:{mime};base64," + base64.b64encode(payload).decode("ascii") + + +def reference_audio_data_url(path: str | None) -> str | None: + """Encode a local reference WAV for a Realtime session update.""" + if path is None: + return None + audio = Path(path).expanduser().resolve() + if not audio.is_file(): + raise FileNotFoundError(f"Reference audio does not exist: {audio}") + return audio_data_url(audio) + + +def chunk_period_ms(events: Sequence[dict[str, object]], *, default: int = 1000) -> int: + """Read the negotiated native-duplex model-unit duration from session events.""" + for event in reversed(events): + session = event.get("session") + capabilities = session.get("capabilities") if isinstance(session, dict) else None + value = capabilities.get("chunk_period_ms") if isinstance(capabilities, dict) else None + if isinstance(value, int) and not isinstance(value, bool) and value > 0: + return value + return default + + +def has_residual_model_unit(pcm16: bytes, *, chunk_period_ms: int) -> bool: + """True when ``pcm16`` (16 kHz mono) does not end on a model-unit boundary.""" + unit_bytes = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * chunk_period_ms // 1000 + return bool(unit_bytes and len(pcm16) % unit_bytes) + + +def read_pcm16_wav(path: Path, *, sample_rate_hz: int = 16_000) -> bytes: + """Read a mono, uncompressed PCM16 WAV file at the expected rate.""" + with wave.open(str(path), "rb") as wav_file: + if wav_file.getnchannels() != 1: + raise ValueError("input WAV must be mono") + if wav_file.getsampwidth() != 2: + raise ValueError("input WAV must be 16-bit PCM") + if wav_file.getframerate() != sample_rate_hz: + raise ValueError(f"input WAV must be {sample_rate_hz} Hz") + if wav_file.getcomptype() != "NONE": + raise ValueError("input WAV must be uncompressed PCM") + return wav_file.readframes(wav_file.getnframes()) + + +def write_pcm16_wav(path: Path, pcm16: bytes, *, sample_rate_hz: int) -> None: + """Write mono PCM16 bytes as a WAV file.""" + with wave.open(str(path), "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(sample_rate_hz) + wav_file.writeframes(pcm16) + + +# --------------------------------------------------------------------------- +# Probe/benchmark helpers + + +def build_realtime_url( + url: str, + model: str | None, + *, + autostart: bool | None = None, + native_duplex: bool | None = None, + session_id: str | None = None, + extra_query: dict[str, str] | None = None, +) -> str: + """Add explicit duplex query parameters to a Realtime URL. + + :class:`DuplexClient` builds its own URL; this helper is for drivers that + speak the wire protocol directly. ``native_duplex`` sets the per-session + model-native opt-in query flag; other model-specific query flags ride in + ``extra_query``. ``http(s)`` URLs are rewritten to ``ws(s)``. + """ + parts = urlsplit(url) + if parts.scheme in {"http", "https"}: + parts = parts._replace(scheme="ws" if parts.scheme == "http" else "wss") + if parts.scheme not in {"ws", "wss"} or not parts.netloc: + raise ValueError(f"Unsupported Realtime URL: {url!r}") + query = dict(parse_qsl(parts.query, keep_blank_values=True)) + query.setdefault("duplex", "1") + if model: + query.setdefault("model", model) + if native_duplex is not None: + query["native_duplex"] = "1" if native_duplex else "0" + for key, value in (extra_query or {}).items(): + query.setdefault(key, value) + if autostart is not None: + query.setdefault("autostart", "1" if autostart else "0") + if session_id: + query.setdefault("session_id", session_id) + return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment)) + + +async def wait_for_condition( + predicate: Callable[[], bool], + *, + timeout_s: float, + label: str, +) -> None: + """Poll a collector predicate without coupling to a scenario runner.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if predicate(): + return + await asyncio.sleep(0.02) + raise TimeoutError(f"Timed out waiting for {label}") + + +def summarize_session_request_metrics( + request_metrics: list[dict[str, object]], + *, + session_id: str | None, +) -> dict[str, object]: + """Average client-observed metrics across turns that emitted audio. + + ``request_metrics`` entries are caller-assembled dicts; keys that are + absent or non-numeric in an entry are simply skipped. ``rtf`` is not + produced by :meth:`EventCollector.timing_summary` (which reports raw data + only) — callers that want ``mean_rtf`` add an ``rtf`` value per turn, + e.g. via ``vllm_omni.metrics.definitions.compute_audio_rtf``. + """ + + def mean(metric: str, *, digits: int = 3) -> float | None: + values = [ + float(request[metric]) + for request in request_metrics + if isinstance(request.get(metric), int | float) and math.isfinite(float(request[metric])) + ] + return round(sum(values) / len(values), digits) if values else None + + return { + "session_id": session_id, + "audio_turn_count": len(request_metrics), + "mean_ttft_ms": mean("ttft_ms"), + "mean_ttfp_ms": mean("ttfp_ms"), + "mean_rtf": mean("rtf", digits=6), + } + + +async def acknowledge_collected_playback(client: DuplexClient, collector: EventCollector) -> None: + """Ack playback of every collected response's audio (probe shorthand). + + A response that has produced no audio yet is still acked (0 ms) unless it + already finished: the ack checkpoints the response's history position on + the server, so a user input committed later cannot displace it. + """ + output_format = AudioFormat("pcm16", collector.output_sample_rate_hz) + for response_id in collector.response_ids: + pcm16 = collector.audio_bytes(response_id) + if not pcm16 and collector.response_is_done(response_id): + continue + played_ms = output_format.duration_ms(len(pcm16)) + await client.ack_playback( + played_ms, + response_id=response_id, + item_id=f"item_{response_id}", + ) diff --git a/vllm_omni/clients/minicpmo_4_5.py b/vllm_omni/clients/minicpmo_4_5.py new file mode 100644 index 000000000..4b25346d4 --- /dev/null +++ b/vllm_omni/clients/minicpmo_4_5.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Client-side duplex session preset for MiniCPM-o 4.5 native duplex.""" + +from __future__ import annotations + +from dataclasses import replace + +from vllm_omni.clients.duplex import SessionConfig + +__all__ = ["create_duplex_session_config"] + + +def create_duplex_session_config(*, ref_audio: str | None = None, **overrides: object) -> SessionConfig: + """Session preset matching the MiniCPM-o 4.5 native duplex deployment.""" + extra_body: dict[str, object] = {"native_duplex": True, "force_listen_count": 0} + extra_body.update(overrides.pop("extra_body", {})) + config = SessionConfig( + ref_audio=ref_audio, + overlap_policy="listen_only", + playback_commit_policy="ack_only", + extra_body=extra_body, + ) + return replace(config, **overrides) if overrides else config diff --git a/vllm_omni/clients/personaplex.py b/vllm_omni/clients/personaplex.py new file mode 100644 index 000000000..a94a6b096 --- /dev/null +++ b/vllm_omni/clients/personaplex.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Client-side duplex session preset for PersonaPlex duplex serving.""" + +from __future__ import annotations + +from dataclasses import replace + +from vllm_omni.clients.duplex import AudioFormat, SessionConfig + +__all__ = ["create_duplex_session_config"] + + +def create_duplex_session_config(*, voice: str = "NATF2.pt", persona: str = "", **overrides: object) -> SessionConfig: + """Session preset matching the PersonaPlex duplex deployment (24 kHz float32 in).""" + config = SessionConfig( + input_audio=AudioFormat("pcm_f32le", 24_000), + output_audio=AudioFormat("pcm16", 24_000), + voice=voice, + instructions=persona, + ) + return replace(config, **overrides) if overrides else config diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index 9e894039f..49393122b 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -22,7 +22,7 @@ import uuid import weakref from collections.abc import Mapping, Sequence from dataclasses import asdict -from typing import TYPE_CHECKING, Any, Literal, cast +from typing import Any, Literal, cast import janus import torch @@ -58,6 +58,13 @@ from vllm_omni.engine.async_engine_utils import ( upgrade_to_omni_request, weak_shutdown_async_omni_engine, ) +from vllm_omni.engine.duplex.control_client import DuplexControlClient +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import ( + load_duplex_runtime_extension, + validate_duplex_runtime_extension, +) from vllm_omni.engine.messages import ( AbortRequestMessage, AbortResultMessage, @@ -90,11 +97,6 @@ from vllm_omni.metrics.prometheus import OmniRequestCounter logger = init_logger(__name__) -if TYPE_CHECKING: - from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlClient - from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity - from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence - _STARTUP_POLL_INTERVAL_S = 1.0 _REQUEST_QUEUE_MAXSIZE = 256 @@ -389,11 +391,6 @@ class AsyncOmniEngine: membership_controller = self._runtime.create_membership_controller() duplex_runtime_extension = None if self._duplex_control_enabled: - from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( - load_duplex_runtime_extension, - validate_duplex_runtime_extension, - ) - duplex_runtime_extension = load_duplex_runtime_extension( getattr(self, "_duplex_runtime_extension_path", None) ) @@ -1812,8 +1809,6 @@ class AsyncOmniEngine: ) def _get_duplex_control_client(self) -> DuplexControlClient: - from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlClient - client = getattr(self, "_duplex_control_client", None) if client is None: transport = getattr(self, "_correlated_rpc_client", None) diff --git a/vllm_omni/engine/duplex/__init__.py b/vllm_omni/engine/duplex/__init__.py new file mode 100644 index 000000000..bd5ae0b3f --- /dev/null +++ b/vllm_omni/engine/duplex/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Engine-side helpers for the full-duplex runtime.""" diff --git a/vllm_omni/experimental/fullduplex/engine/contracts.py b/vllm_omni/engine/duplex/contracts.py similarity index 98% rename from vllm_omni/experimental/fullduplex/engine/contracts.py rename to vllm_omni/engine/duplex/contracts.py index ae449e82f..6af4fdea5 100644 --- a/vllm_omni/experimental/fullduplex/engine/contracts.py +++ b/vllm_omni/engine/duplex/contracts.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Model-neutral contracts for the experimental duplex engine plugin. @@ -17,8 +17,8 @@ from enum import Enum from types import MappingProxyType from typing import Any, Protocol +from vllm_omni.engine.duplex.messages import DuplexFence from vllm_omni.engine.messages import EngineQueueMessage -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence class SessionMode(str, Enum): diff --git a/vllm_omni/experimental/fullduplex/engine/duplex_control_client.py b/vllm_omni/engine/duplex/control_client.py similarity index 96% rename from vllm_omni/experimental/fullduplex/engine/duplex_control_client.py rename to vllm_omni/engine/duplex/control_client.py index b37effe32..05ec46e49 100644 --- a/vllm_omni/experimental/fullduplex/engine/duplex_control_client.py +++ b/vllm_omni/engine/duplex/control_client.py @@ -1,14 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import uuid from collections.abc import Callable -from vllm_omni.experimental.fullduplex.engine.contracts import CorrelatedRpcTransport -from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity -from vllm_omni.experimental.fullduplex.engine.messages import ( +from vllm_omni.engine.duplex.contracts import CorrelatedRpcTransport +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity +from vllm_omni.engine.duplex.messages import ( AppendDuplexInputMessage, CloseDuplexSessionMessage, DuplexControlResultMessage, diff --git a/vllm_omni/experimental/fullduplex/engine/duplex_control_plane.py b/vllm_omni/engine/duplex/control_plane.py similarity index 99% rename from vllm_omni/experimental/fullduplex/engine/duplex_control_plane.py rename to vllm_omni/engine/duplex/control_plane.py index a1d3dac00..b425542b8 100644 --- a/vllm_omni/experimental/fullduplex/engine/duplex_control_plane.py +++ b/vllm_omni/engine/duplex/control_plane.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Orchestrator-side control plane for experimental duplex sessions. @@ -22,7 +22,7 @@ from typing import Any, Protocol from vllm.logger import init_logger -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex.contracts import ( DuplexAppendPlan, DuplexInputMode, DuplexOutputContext, @@ -37,15 +37,8 @@ from vllm_omni.experimental.fullduplex.engine.contracts import ( SessionMode, duplex_resource_request_id, ) -from vllm_omni.experimental.fullduplex.engine.duplex_session import ( - DuplexAppendReservation, - DuplexFenceMismatchError, - DuplexSessionExpiry, - DuplexSessionRuntimeManager, - DuplexSessionRuntimeState, -) -from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity, DuplexLeaseConfig -from vllm_omni.experimental.fullduplex.engine.messages import ( +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity, DuplexLeaseConfig +from vllm_omni.engine.duplex.messages import ( AppendDuplexInputMessage, CloseDuplexSessionMessage, DuplexControlError, @@ -57,6 +50,13 @@ from vllm_omni.experimental.fullduplex.engine.messages import ( SignalDuplexTurnMessage, TouchDuplexSessionMessage, ) +from vllm_omni.engine.duplex.session import ( + DuplexAppendReservation, + DuplexFenceMismatchError, + DuplexSessionExpiry, + DuplexSessionRuntimeManager, + DuplexSessionRuntimeState, +) logger = init_logger(__name__) diff --git a/vllm_omni/experimental/fullduplex/engine/intermediate.py b/vllm_omni/engine/duplex/intermediate.py similarity index 96% rename from vllm_omni/experimental/fullduplex/engine/intermediate.py rename to vllm_omni/engine/duplex/intermediate.py index d85d8f712..3beb73099 100644 --- a/vllm_omni/experimental/fullduplex/engine/intermediate.py +++ b/vllm_omni/engine/duplex/intermediate.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations from typing import TypedDict diff --git a/vllm_omni/experimental/fullduplex/engine/lease.py b/vllm_omni/engine/duplex/lease.py similarity index 96% rename from vllm_omni/experimental/fullduplex/engine/lease.py rename to vllm_omni/engine/duplex/lease.py index 09430a7bb..c38faacd1 100644 --- a/vllm_omni/experimental/fullduplex/engine/lease.py +++ b/vllm_omni/engine/duplex/lease.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Generic engine lease primitives shared by optional session runtimes.""" @@ -8,7 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field from enum import Enum -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.engine.duplex.messages import DuplexFence class DuplexLeaseActivity(str, Enum): diff --git a/vllm_omni/experimental/fullduplex/engine/messages.py b/vllm_omni/engine/duplex/messages.py similarity index 97% rename from vllm_omni/experimental/fullduplex/engine/messages.py rename to vllm_omni/engine/duplex/messages.py index 94f154a8a..2c49d573f 100644 --- a/vllm_omni/experimental/fullduplex/engine/messages.py +++ b/vllm_omni/engine/duplex/messages.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Queue contracts owned by the experimental duplex control plane.""" diff --git a/vllm_omni/experimental/fullduplex/engine/duplex_runtime.py b/vllm_omni/engine/duplex/runtime.py similarity index 91% rename from vllm_omni/experimental/fullduplex/engine/duplex_runtime.py rename to vllm_omni/engine/duplex/runtime.py index acd977448..dede0e0a6 100644 --- a/vllm_omni/experimental/fullduplex/engine/duplex_runtime.py +++ b/vllm_omni/engine/duplex/runtime.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Experimental duplex runtime loading and compatibility exports.""" @@ -7,7 +7,7 @@ from __future__ import annotations from importlib import import_module -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex.contracts import ( DuplexAppendPlan, DuplexInputMode, DuplexOutputAction, @@ -63,7 +63,13 @@ def validate_duplex_runtime_extension( return typed_extension -from vllm_omni.experimental.fullduplex.engine.duplex_session import ( # noqa: E402, F401 +from vllm_omni.engine.duplex.lease import ( # noqa: E402, F401 + DuplexLeaseActivity, + DuplexLeaseConfig, + DuplexLeaseState, + DuplexSessionExpiry, +) +from vllm_omni.engine.duplex.session import ( # noqa: E402, F401 DuplexAppendReservation, DuplexCompletedAppend, DuplexFenceMismatchError, @@ -73,12 +79,6 @@ from vllm_omni.experimental.fullduplex.engine.duplex_session import ( # noqa: E DuplexSessionRuntimeState, DuplexStageBinding, ) -from vllm_omni.experimental.fullduplex.engine.lease import ( # noqa: E402, F401 - DuplexLeaseActivity, - DuplexLeaseConfig, - DuplexLeaseState, - DuplexSessionExpiry, -) __all__ = [ "DuplexAppendPlan", diff --git a/vllm_omni/experimental/fullduplex/engine/duplex_session.py b/vllm_omni/engine/duplex/session.py similarity index 98% rename from vllm_omni/experimental/fullduplex/engine/duplex_session.py rename to vllm_omni/engine/duplex/session.py index 244005f06..b7ad51504 100644 --- a/vllm_omni/experimental/fullduplex/engine/duplex_session.py +++ b/vllm_omni/engine/duplex/session.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -9,17 +9,17 @@ from collections.abc import Callable, Iterable from dataclasses import dataclass, field from typing import Any -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex.contracts import ( DuplexInputMode, DuplexRuntimeCapabilities, ) -from vllm_omni.experimental.fullduplex.engine.lease import ( +from vllm_omni.engine.duplex.lease import ( DuplexLeaseActivity, DuplexLeaseConfig, DuplexLeaseState, DuplexSessionExpiry, ) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence +from vllm_omni.engine.duplex.messages import DuplexFence def _default_capabilities() -> DuplexRuntimeCapabilities: diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index bda3875c2..e86dc7673 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -21,7 +21,7 @@ import time as _time from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from http import HTTPStatus -from typing import TYPE_CHECKING, Any +from typing import Any import janus import torch @@ -38,6 +38,23 @@ from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig from vllm_omni.distributed.omni_connectors.utils.config import stage_receives_chunks from vllm_omni.engine import OmniEngineCoreRequest from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker +from vllm_omni.engine.duplex.contracts import ( + DuplexControlPlanePort, + DuplexOutputContext, + DuplexOutputDecision, + DuplexRequestIdentity, + DuplexRuntimeExtension, + DuplexStageRequestContext, + DuplexStageSubmission, + DuplexStageSubmissionResult, +) +from vllm_omni.engine.duplex.control_plane import DuplexControlPlane +from vllm_omni.engine.duplex.lease import DuplexLeaseConfig +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.session import ( + DuplexSessionRuntimeManager, + DuplexSessionRuntimeState, +) from vllm_omni.engine.membership_controller import MembershipController from vllm_omni.engine.messages import ( AbortRequestMessage, @@ -64,6 +81,7 @@ from vllm_omni.metrics.prometheus import OmniRequestCounter from vllm_omni.metrics.stat_logger import OmniPrometheusStatLogger from vllm_omni.metrics.utils import DIFFUSION_METRICS_ONLY_REQUEST_ID from vllm_omni.outputs import OmniRequestOutput +from vllm_omni.outputs.duplex import attach_duplex_output_decision logger = init_logger(__name__) @@ -90,24 +108,6 @@ def _event_driven_orch_enabled() -> bool: return os.environ.get(_EVENT_DRIVEN_ORCH_ENV, "0").strip().lower() in ("1", "true", "yes", "on") -if TYPE_CHECKING: - from vllm_omni.experimental.fullduplex.engine.contracts import ( - DuplexControlPlanePort, - DuplexOutputContext, - DuplexOutputDecision, - DuplexRequestIdentity, - DuplexRuntimeExtension, - DuplexStageRequestContext, - DuplexStageSubmission, - DuplexStageSubmissionResult, - ) - from vllm_omni.experimental.fullduplex.engine.duplex_session import ( - DuplexSessionRuntimeManager, - DuplexSessionRuntimeState, - ) - from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence - - def _build_terminal_empty_output( request_id: str, *, @@ -342,8 +342,6 @@ class _OrchestratorDuplexStagePort: ) def ensure_request(self, context: DuplexStageRequestContext) -> None: - from vllm_omni.experimental.fullduplex.engine.contracts import DuplexRequestIdentity - request_state = self._request_states.get(context.request_id) if request_state is None: request_state = OrchestratorRequestState( @@ -365,8 +363,6 @@ class _OrchestratorDuplexStagePort: self._sync_bridge_state(request_state, context) async def submit(self, submission: DuplexStageSubmission) -> DuplexStageSubmissionResult: - from vllm_omni.experimental.fullduplex.engine.contracts import DuplexStageSubmissionResult - context = submission.context request_state = self._request_states.get(context.request_id) if request_state is None: @@ -491,9 +487,6 @@ class Orchestrator: self.duplex_control_plane: DuplexControlPlanePort | None = None self._duplex_reaper_interval_s = 1.0 if enable_duplex_control: - from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import DuplexControlPlane - from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseConfig - runtime_session_config = duplex_session_config or DuplexSessionRuntimeConfig() self._duplex_reaper_interval_s = runtime_session_config.reaper_interval_s @@ -2197,11 +2190,6 @@ class Orchestrator: identity = req_state.duplex_identity if identity is None: return None - from vllm_omni.experimental.fullduplex.engine.contracts import ( - DuplexOutputContext, - DuplexRequestIdentity, - ) - fence = req_state.duplex_stage_fences.get(stage_id, identity.fence) if stage_id is not None else identity.fence segment = req_state.streaming.segment(stage_id) return DuplexOutputContext( @@ -2256,8 +2244,6 @@ class Orchestrator: action = getattr(decision.action, "value", decision.action) if action != "direct_response": raise ValueError(f"Unsupported duplex output action: {action}") - from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision - engine_output = attach_duplex_output_decision( OmniRequestOutput.from_stage_output( output, diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 8ac0fde56..4026907e1 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ AsyncOmni - Refactored async orchestrator using AsyncOmniEngine. @@ -27,8 +30,17 @@ from vllm.utils import random_uuid from vllm.v1.engine.exceptions import EngineDeadError from vllm_omni.diffusion.data import CuMemTag, OmniACK, OmniSleepTask, OmniWakeTask +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity +from vllm_omni.engine.duplex.messages import ( + DuplexFence, + DuplexSessionLifecycleMessage, +) from vllm_omni.engine.messages import ErrorMessage, OutputMessage from vllm_omni.entrypoints.client_request_state import ClientRequestState +from vllm_omni.entrypoints.duplex_request_client import ( + DuplexRequestClient, + DuplexRequestOutputPort, +) from vllm_omni.entrypoints.omni_base import ( OmniBase, OmniEngineDeadError, @@ -43,12 +55,6 @@ if TYPE_CHECKING: from vllm.tokenizers import TokenizerLike from vllm.v1.engine import PauseMode - from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity - from vllm_omni.experimental.fullduplex.engine.messages import ( - DuplexFence, - DuplexSessionLifecycleMessage, - ) - from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient from vllm_omni.inputs.data import OmniInteractionPrompt, OmniPromptType logger = init_logger(__name__) @@ -410,11 +416,6 @@ class AsyncOmni(EngineClient, OmniBase): ) def _get_duplex_request_client(self) -> DuplexRequestClient: - from vllm_omni.experimental.fullduplex.request_client import ( - DuplexRequestClient, - DuplexRequestOutputPort, - ) - client = getattr(self, "_duplex_request_client", None) if client is None: engine = getattr(self, "engine", None) @@ -433,8 +434,6 @@ class AsyncOmni(EngineClient, OmniBase): @staticmethod def _duplex_data_plane_request_info(result: dict[str, object]) -> tuple[str | None, int | None]: - from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient - return DuplexRequestClient.request_info(result) async def _collect_duplex_data_plane_outputs( @@ -454,14 +453,10 @@ class AsyncOmni(EngineClient, OmniBase): @classmethod def _is_direct_duplex_data_plane_response(cls, output: object) -> bool: - from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient - return DuplexRequestClient.is_direct_response(output) @classmethod def _duplex_multimodal_output(cls, output: object) -> dict[str, object]: - from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient - return DuplexRequestClient.multimodal_output(output) # ==================== Generate Method ==================== diff --git a/vllm_omni/entrypoints/cli/benchmark/serve.py b/vllm_omni/entrypoints/cli/benchmark/serve.py index 7c7b05221..683f0ea84 100644 --- a/vllm_omni/entrypoints/cli/benchmark/serve.py +++ b/vllm_omni/entrypoints/cli/benchmark/serve.py @@ -13,7 +13,6 @@ import argparse from vllm.benchmarks.serve import add_cli_args -from vllm_omni.benchmarks.serve import main from vllm_omni.entrypoints.cli.benchmark.base import OmniBenchmarkSubcommandBase from vllm_omni.entrypoints.cli.benchmark.cli_args import ( add_omni_args, @@ -38,5 +37,11 @@ class OmniBenchmarkServingSubcommand(OmniBenchmarkSubcommandBase): @staticmethod def cmd(args: argparse.Namespace) -> None: + # Deferred: the benchmark runtime imports the duplex client library + # (vllm_omni.clients), which must stay out of the CLI import graph — + # this module is imported by `vllm-omni serve` as well + # (tests/engine/test_duplex_import_boundary.py enforces the boundary). + from vllm_omni.benchmarks.serve import main + preprocess_serve_args(args) main(args) diff --git a/vllm_omni/experimental/fullduplex/openai/__init__.py b/vllm_omni/entrypoints/duplex/__init__.py similarity index 61% rename from vllm_omni/experimental/fullduplex/openai/__init__.py rename to vllm_omni/entrypoints/duplex/__init__.py index 1bb45f8d4..736e98a6a 100644 --- a/vllm_omni/experimental/fullduplex/openai/__init__.py +++ b/vllm_omni/entrypoints/duplex/__init__.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from .websocket import DuplexWebSocketActor diff --git a/vllm_omni/experimental/fullduplex/openai/audio.py b/vllm_omni/entrypoints/duplex/audio.py similarity index 99% rename from vllm_omni/experimental/fullduplex/openai/audio.py rename to vllm_omni/entrypoints/duplex/audio.py index 023bfdd9e..644a7a297 100644 --- a/vllm_omni/experimental/fullduplex/openai/audio.py +++ b/vllm_omni/entrypoints/duplex/audio.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations diff --git a/vllm_omni/entrypoints/openai/duplex_capability.py b/vllm_omni/entrypoints/duplex/capability.py similarity index 93% rename from vllm_omni/entrypoints/openai/duplex_capability.py rename to vllm_omni/entrypoints/duplex/capability.py index c3e4e3e38..898daec10 100644 --- a/vllm_omni/entrypoints/openai/duplex_capability.py +++ b/vllm_omni/entrypoints/duplex/capability.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from vllm.logger import init_logger diff --git a/vllm_omni/experimental/fullduplex/openai/chat_fallback.py b/vllm_omni/entrypoints/duplex/chat_fallback.py similarity index 98% rename from vllm_omni/experimental/fullduplex/openai/chat_fallback.py rename to vllm_omni/entrypoints/duplex/chat_fallback.py index 4edba1646..31df63833 100644 --- a/vllm_omni/experimental/fullduplex/openai/chat_fallback.py +++ b/vllm_omni/entrypoints/duplex/chat_fallback.py @@ -12,7 +12,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.logger import init_logger -from vllm_omni.experimental.fullduplex.openai.protocol import DuplexSession +from vllm_omni.entrypoints.duplex.protocol import DuplexSession logger = init_logger(__name__) @@ -94,6 +94,7 @@ class ChatFallbackProjectorMixin: if response_config.max_tokens is not None: kwargs["max_tokens"] = response_config.max_tokens model_extra = dict(response_config.extra_body) + model_extra.pop("native_duplex", None) model_extra.pop("minicpmo45_native_duplex", None) tools = model_extra.pop("realtime_response_tools", model_extra.pop("realtime_tools", None)) tool_choice = model_extra.pop( diff --git a/vllm_omni/experimental/fullduplex/openai/commit_policy.py b/vllm_omni/entrypoints/duplex/commit_policy.py similarity index 91% rename from vllm_omni/experimental/fullduplex/openai/commit_policy.py rename to vllm_omni/entrypoints/duplex/commit_policy.py index 940509ec3..868b98c98 100644 --- a/vllm_omni/experimental/fullduplex/openai/commit_policy.py +++ b/vllm_omni/entrypoints/duplex/commit_policy.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations from dataclasses import dataclass diff --git a/vllm_omni/experimental/fullduplex/openai/protocol.py b/vllm_omni/entrypoints/duplex/protocol.py similarity index 97% rename from vllm_omni/experimental/fullduplex/openai/protocol.py rename to vllm_omni/entrypoints/duplex/protocol.py index 8b3812bc3..9fd3b0ef5 100644 --- a/vllm_omni/experimental/fullduplex/openai/protocol.py +++ b/vllm_omni/entrypoints/duplex/protocol.py @@ -27,6 +27,30 @@ class DuplexSessionState(str, Enum): CLOSED = "closed" +# Canonical extra_body key for the per-session opt-in to a model-native +# duplex runtime, plus the deprecated model-prefixed spelling it replaced. +# Client payloads may still use the legacy key; it is folded into the +# canonical one at ingestion so everything downstream (and every echo in +# session.created/session.updated) sees only NATIVE_DUPLEX_KEY. +NATIVE_DUPLEX_KEY = "native_duplex" +LEGACY_NATIVE_DUPLEX_KEY = "minicpmo45_native_duplex" + + +def normalize_native_duplex_key(extra_body: dict[str, object]) -> dict[str, object]: + """Fold the deprecated alias into the canonical key (canonical wins).""" + if LEGACY_NATIVE_DUPLEX_KEY in extra_body: + legacy = extra_body.pop(LEGACY_NATIVE_DUPLEX_KEY) + extra_body.setdefault(NATIVE_DUPLEX_KEY, legacy) + return extra_body + + +def native_duplex_opt_in(extra_body: Mapping[str, object]) -> object: + """Read the opt-in flag, accepting the deprecated alias for raw dicts.""" + if NATIVE_DUPLEX_KEY in extra_body: + return extra_body[NATIVE_DUPLEX_KEY] + return extra_body.get(LEGACY_NATIVE_DUPLEX_KEY) + + class DuplexTurnState(str, Enum): IDLE = "idle" USER_SPEAKING = "user_speaking" @@ -98,44 +122,6 @@ class DuplexCapabilities: chunk_period_ms: int | None = 1000 target_barge_in_latency_ms: int | None = 1000 - @classmethod - def minicpmo45_native(cls, *, max_sessions: int = 1) -> DuplexCapabilities: - supports_multi_session = max_sessions > 1 - return cls( - supports_model_native_turn_policy=True, - supports_barge_in=True, - supports_input_append=True, - supports_replace_latest_chunk=False, - supports_reencode_context=False, - supports_turn_commit_only=False, - supports_kv_lease=False, - supports_core_kv_lease=False, - supports_model_internal_state=True, - supports_stage_resumption=True, - supports_scheduler_native_append=False, - supports_core_resumable_request=True, - supports_stage_connector_handoff=True, - supports_independent_io_streams=True, - supports_realtime_endpoint=True, - supports_multi_session=supports_multi_session, - supports_multi_session_same_replica=supports_multi_session, - supports_session_lease=True, - supports_session_resume=True, - session_admission_mode="engine_managed", - supports_audio_truncate=True, - requires_model_runner_kv=True, - requires_native_stage_role=True, - implementation_level="model_native_duplex", - adapter_patterns=["scheduler_data_plane"], - input_modes=["append_audio_chunk"], - signal_sources=["model_native", "client_event", "server_policy"], - stage_handoff_transport="scheduler_data_plane", - chunk_period_ms=1000, - # Barge-in latency depends on client chunking and lacks hardware E2E - # measurement, so do not advertise an invented target. - target_barge_in_latency_ms=None, - ) - def as_dict(self) -> dict[str, object]: return { "supports_session_adapter": self.supports_session_adapter, @@ -310,7 +296,7 @@ class DuplexSessionConfig: if isinstance(source.get("modalities"), list) and all(isinstance(x, str) for x in source["modalities"]): config.modalities = list(source["modalities"]) if isinstance(source.get("extra_body"), dict): - config.extra_body = dict(source["extra_body"]) + config.extra_body = normalize_native_duplex_key(dict(source["extra_body"])) extra = config.extra_body if isinstance(extra.get("overlap_policy"), str): config.overlap_policy = cls._normalize_overlap_policy(extra["overlap_policy"]) diff --git a/vllm_omni/experimental/fullduplex/openai/realtime_input.py b/vllm_omni/entrypoints/duplex/realtime_input.py similarity index 99% rename from vllm_omni/experimental/fullduplex/openai/realtime_input.py rename to vllm_omni/entrypoints/duplex/realtime_input.py index 3d77a4172..03710b43b 100644 --- a/vllm_omni/experimental/fullduplex/openai/realtime_input.py +++ b/vllm_omni/entrypoints/duplex/realtime_input.py @@ -12,15 +12,16 @@ from uuid import uuid4 import numpy as np -from vllm_omni.experimental.fullduplex.openai.audio import ( +from vllm_omni.entrypoints.duplex.audio import ( convert_input_audio_with_rate, ) -from vllm_omni.experimental.fullduplex.openai.realtime_state import ( +from vllm_omni.entrypoints.duplex.protocol import normalize_native_duplex_key +from vllm_omni.entrypoints.duplex.realtime_state import ( REALTIME_INPUT_AUDIO_FORMATS, REALTIME_OUTPUT_AUDIO_FORMATS, RealtimeStateOwner, ) -from vllm_omni.experimental.fullduplex.openai.vad import ( +from vllm_omni.entrypoints.duplex.vad import ( SILERO_VAD_MIN_THRESHOLD, ServerVADUnavailableError, SileroStreamingVAD, @@ -752,7 +753,9 @@ class RealtimeInputTranslator(RealtimeStateOwner): audio_input = audio_config.get("input") if isinstance(audio_config, dict) else None audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None extra_body_payload = session_payload.get("extra_body") - extra_body = dict(extra_body_payload) if isinstance(extra_body_payload, dict) else {} + extra_body = ( + normalize_native_duplex_key(dict(extra_body_payload)) if isinstance(extra_body_payload, dict) else {} + ) extra_body["realtime_session_payload"] = self._json_safe_realtime_payload(session_payload) if isinstance(session_payload.get("tools"), list): extra_body["realtime_tools"] = session_payload["tools"] diff --git a/vllm_omni/experimental/fullduplex/openai/realtime_output.py b/vllm_omni/entrypoints/duplex/realtime_output.py similarity index 97% rename from vllm_omni/experimental/fullduplex/openai/realtime_output.py rename to vllm_omni/entrypoints/duplex/realtime_output.py index 2a68f79ad..e8c99f218 100644 --- a/vllm_omni/experimental/fullduplex/openai/realtime_output.py +++ b/vllm_omni/entrypoints/duplex/realtime_output.py @@ -1,10 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations from typing import Any from uuid import uuid4 -from vllm_omni.experimental.fullduplex.openai.audio import convert_output_audio -from vllm_omni.experimental.fullduplex.openai.realtime_state import ( +from vllm_omni.entrypoints.duplex.audio import convert_output_audio +from vllm_omni.entrypoints.duplex.realtime_state import ( _RealtimeResponseState, ) @@ -72,18 +75,25 @@ class RealtimeOutputProjector: payloads.extend(self._ensure_response_audio_part_added(response_id)) return payloads if event_type == "response.listen": - return [ - { - "type": "response.listen", - "session_id": event.get("session_id"), - "epoch": event.get("epoch"), - "response": { - "object": "realtime.response", - "status": "listening", - "metadata": event, - }, - } - ] + listen_response: dict[str, object] = { + "object": "realtime.response", + "status": "listening", + "metadata": event, + } + listen_payload: dict[str, object] = { + "type": "response.listen", + "session_id": event.get("session_id"), + "epoch": event.get("epoch"), + "response": listen_response, + } + listen_response_id = event.get("response_id") + if isinstance(listen_response_id, str) and listen_response_id: + # Keep the response identity visible on the Realtime dialect: + # a listen that terminates a precreated response is matched to + # it by this id (clients demultiplex on response_id). + listen_response["id"] = listen_response_id + listen_payload["response_id"] = listen_response_id + return [listen_payload] if event_type == "response.speak": response_id = event.get("response_id") state = self._response_state(response_id) diff --git a/vllm_omni/experimental/fullduplex/openai/realtime_session.py b/vllm_omni/entrypoints/duplex/realtime_session.py similarity index 95% rename from vllm_omni/experimental/fullduplex/openai/realtime_session.py rename to vllm_omni/entrypoints/duplex/realtime_session.py index a85ab1354..c4604ff39 100644 --- a/vllm_omni/experimental/fullduplex/openai/realtime_session.py +++ b/vllm_omni/entrypoints/duplex/realtime_session.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import json @@ -6,11 +9,11 @@ from uuid import uuid4 from fastapi import WebSocket -from vllm_omni.experimental.fullduplex.openai import realtime_state as _state -from vllm_omni.experimental.fullduplex.openai.realtime_input import ( +from vllm_omni.entrypoints.duplex import realtime_state as _state +from vllm_omni.entrypoints.duplex.realtime_input import ( RealtimeInputTranslator, ) -from vllm_omni.experimental.fullduplex.openai.realtime_output import ( +from vllm_omni.entrypoints.duplex.realtime_output import ( RealtimeOutputProjector, ) diff --git a/vllm_omni/experimental/fullduplex/openai/realtime_state.py b/vllm_omni/entrypoints/duplex/realtime_state.py similarity index 94% rename from vllm_omni/experimental/fullduplex/openai/realtime_state.py rename to vllm_omni/entrypoints/duplex/realtime_state.py index 6a2c8f347..4a1e7bdba 100644 --- a/vllm_omni/experimental/fullduplex/openai/realtime_state.py +++ b/vllm_omni/entrypoints/duplex/realtime_state.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import asyncio @@ -133,12 +136,18 @@ class RealtimeSessionState: if hasattr(query_params, "query_params"): query_params = query_params.query_params getter = query_params.get if hasattr(query_params, "get") else None - native_duplex = getter("minicpmo45_native_duplex") if getter is not None else None + # Canonical query param plus the deprecated model-prefixed alias; + # both seed the canonical extra_body key. + native_duplex = None + if getter is not None: + native_duplex = getter("native_duplex") + if native_duplex is None: + native_duplex = getter("minicpmo45_native_duplex") resume_only = getter("resume") if getter is not None else None autostart = getter("autostart") if getter is not None else None default_extra_body = {} if str(native_duplex).strip().lower() in {"1", "true", "yes", "on"}: - default_extra_body["minicpmo45_native_duplex"] = True + default_extra_body["native_duplex"] = True return cls( _resume_only=( str(resume_only).strip().lower() in {"1", "true", "yes", "on"} diff --git a/vllm_omni/experimental/fullduplex/openai/runtime_adapter.py b/vllm_omni/entrypoints/duplex/runtime_adapter.py similarity index 98% rename from vllm_omni/experimental/fullduplex/openai/runtime_adapter.py rename to vllm_omni/entrypoints/duplex/runtime_adapter.py index d05b709ff..8bf23e1ed 100644 --- a/vllm_omni/experimental/fullduplex/openai/runtime_adapter.py +++ b/vllm_omni/entrypoints/duplex/runtime_adapter.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import asyncio diff --git a/vllm_omni/experimental/fullduplex/openai/runtime_bridge.py b/vllm_omni/entrypoints/duplex/runtime_bridge.py similarity index 97% rename from vllm_omni/experimental/fullduplex/openai/runtime_bridge.py rename to vllm_omni/entrypoints/duplex/runtime_bridge.py index 98e0aaaf5..6db5be236 100644 --- a/vllm_omni/experimental/fullduplex/openai/runtime_bridge.py +++ b/vllm_omni/entrypoints/duplex/runtime_bridge.py @@ -13,14 +13,14 @@ from enum import Enum import numpy as np from vllm.logger import init_logger -from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlRequestError -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_data_plane_request_info -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.openai.protocol import ( +from vllm_omni.engine.duplex.control_client import DuplexControlRequestError +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import duplex_data_plane_request_info +from vllm_omni.entrypoints.duplex.protocol import ( DuplexSession, DuplexSessionState, ) -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( +from vllm_omni.entrypoints.duplex.runtime_adapter import ( coerce_int, payload_turn_id, ) @@ -372,15 +372,18 @@ class NativeRuntimeBridgeMixin: if expected_epoch is not None and response_epoch != expected_epoch: return native.clear_continuation() - await send_json( - { - "type": "response.listen", - "session_id": session.session_id, - "epoch": response_epoch, - "reason": "continuation_limit", - "model_listen": False, - } - ) + payload = { + "type": "response.listen", + "session_id": session.session_id, + "epoch": response_epoch, + "reason": "continuation_limit", + "model_listen": False, + } + if response_id is not None: + # Clients demultiplex a listen against a precreated response by + # its id; every listen that terminates a response must carry it. + payload["response_id"] = response_id + await send_json(payload) if response_id is None: if session.epoch == response_epoch and response_turn_id is not None: session.complete_model_turn(response_turn_id) @@ -928,6 +931,8 @@ class NativeRuntimeBridgeMixin: if native_result.get("is_buffering") is True or native_result.get("prefill_success") is False: if native_result.get("data_plane_request_id") == session.active_request_id: session.clear_request() + # No response_id here: a buffering listen is informational, not a + # response terminal — clients must not close a handle over it. payload = { "type": "response.listen", "session_id": session.session_id, @@ -986,6 +991,11 @@ class NativeRuntimeBridgeMixin: "reason": native_result.get("reason") or "model_listen", "model_listen": model_listen, } + if response_id is not None: + # The listen is this response's terminal decision (a + # response.done follows below with the same id); carry the id + # so clients can bind the decision to the precreated response. + payload["response_id"] = response_id self._attach_native_runtime_metadata(payload, native_result) await send_json(payload) if native_result.get("abort_data_plane_request") is True and isinstance(data_plane_request_id, str): diff --git a/vllm_omni/experimental/fullduplex/openai/serving.py b/vllm_omni/entrypoints/duplex/serving.py similarity index 97% rename from vllm_omni/experimental/fullduplex/openai/serving.py rename to vllm_omni/entrypoints/duplex/serving.py index 0b3bd246c..d1a028bd4 100644 --- a/vllm_omni/experimental/fullduplex/openai/serving.py +++ b/vllm_omni/entrypoints/duplex/serving.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import asyncio @@ -15,15 +18,16 @@ from fastapi import WebSocket from vllm.logger import init_logger from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig -from vllm_omni.entrypoints.openai.duplex_capability import ( +from vllm_omni.engine.duplex.messages import DuplexFence, DuplexSessionLifecycleMessage +from vllm_omni.engine.duplex.runtime import duplex_resource_request_id +from vllm_omni.entrypoints.duplex.capability import ( should_enable_duplex_endpoint, ) -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence, DuplexSessionLifecycleMessage -from vllm_omni.experimental.fullduplex.openai.chat_fallback import ( +from vllm_omni.entrypoints.duplex.chat_fallback import ( ChatFallbackProjectorMixin, ) -from vllm_omni.experimental.fullduplex.openai.protocol import ( +from vllm_omni.entrypoints.duplex.protocol import ( + NATIVE_DUPLEX_KEY, DuplexCapabilities, DuplexCommittedInput, DuplexOverlapPolicy, @@ -34,30 +38,32 @@ from vllm_omni.experimental.fullduplex.openai.protocol import ( DuplexSessionState, DuplexTurnController, ResponseCreateOptions, + native_duplex_opt_in, + normalize_native_duplex_key, ) -from vllm_omni.experimental.fullduplex.openai.realtime_session import ( +from vllm_omni.entrypoints.duplex.realtime_session import ( REALTIME_OUTPUT_AUDIO_FORMATS, NativeRealtimeSessionProtocol, ) -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( +from vllm_omni.entrypoints.duplex.runtime_adapter import ( ServingRuntimeAdapter, ServingRuntimeConfigError, ServingRuntimeSessionState, load_serving_runtime_adapter, validate_serving_runtime_adapter, ) -from vllm_omni.experimental.fullduplex.openai.runtime_bridge import ( +from vllm_omni.entrypoints.duplex.runtime_bridge import ( NativeRuntimeBridgeMixin, ) -from vllm_omni.experimental.fullduplex.openai.session_attachment import ( +from vllm_omni.entrypoints.duplex.session_attachment import ( DuplexJournalGapError, DuplexSessionAttachmentRegistry, InvalidResumeTokenError, ) -from vllm_omni.experimental.fullduplex.openai.session_runner import ( +from vllm_omni.entrypoints.duplex.session_runner import ( DuplexSessionRunnerMixin, ) -from vllm_omni.experimental.fullduplex.openai.websocket import ( +from vllm_omni.entrypoints.duplex.websocket import ( DOMAIN_TERMINAL_EVENTS, DuplexSessionTasks, DuplexWebSocketActor, @@ -791,18 +797,6 @@ class OmniDuplexSessionHandler( def _runtime_session_state(self, session: DuplexSession) -> ServingRuntimeSessionState: return self._serving_runtime_adapter.session_state(session.session_id) - # Temporary compatibility accessors for downstream tests and extensions. - def _minicpmo_session_state(self, session: DuplexSession) -> ServingRuntimeSessionState: - return self._runtime_session_state(session) - - @property - def _minicpmo_sessions(self): - return self._serving_runtime_adapter.session_states - - @property - def _minicpmo_data_plane(self): - return self._serving_runtime_adapter.data_plane - def _should_force_listen_for_auto_response_overlap( self, session: DuplexSession, @@ -1212,7 +1206,7 @@ class OmniDuplexSessionHandler( "code": "instructions_update_unsupported", "error": "session.update cannot change instructions after the native duplex context is initialized", } - if session.config.extra_body.get("minicpmo45_native_duplex") is not True: + if native_duplex_opt_in(session.config.extra_body) is not True: return None if not self._config_requests_audio_output(candidate_config): return None @@ -1230,7 +1224,7 @@ class OmniDuplexSessionHandler( return ( session.capabilities.implementation_level == "model_native_duplex" and session.capabilities.supports_input_append - and session.config.extra_body.get("minicpmo45_native_duplex") is not False + and native_duplex_opt_in(session.config.extra_body) is not False ) @staticmethod @@ -1385,17 +1379,19 @@ class OmniDuplexSessionHandler( "error": "session.update cannot change ref_audio after the native duplex runtime is open", } extra_body_payload = payload.get("extra_body") - if isinstance(extra_body_payload, dict) and "minicpmo45_native_duplex" in extra_body_payload: - current_native_duplex = session.config.extra_body.get("minicpmo45_native_duplex") - if ( - current_native_duplex is not None - and extra_body_payload["minicpmo45_native_duplex"] != current_native_duplex - ): + if isinstance(extra_body_payload, dict): + # Fold the deprecated model-prefixed alias into the canonical + # native_duplex key before comparing or merging. + extra_body_payload = normalize_native_duplex_key(dict(extra_body_payload)) + payload = {**payload, "extra_body": extra_body_payload} + if isinstance(extra_body_payload, dict) and NATIVE_DUPLEX_KEY in extra_body_payload: + current_native_duplex = native_duplex_opt_in(session.config.extra_body) + if current_native_duplex is not None and extra_body_payload[NATIVE_DUPLEX_KEY] != current_native_duplex: return { "type": "error", "session_id": session.session_id, "code": "native_duplex_mode_update_unsupported", - "error": "session.update cannot change minicpmo45_native_duplex after the session is created", + "error": "session.update cannot change native_duplex after the session is created", } if isinstance(payload.get("instructions"), str): session.config.instructions = str(payload["instructions"]) diff --git a/vllm_omni/experimental/fullduplex/openai/session_attachment.py b/vllm_omni/entrypoints/duplex/session_attachment.py similarity index 99% rename from vllm_omni/experimental/fullduplex/openai/session_attachment.py rename to vllm_omni/entrypoints/duplex/session_attachment.py index b0faa65d2..d68fb268b 100644 --- a/vllm_omni/experimental/fullduplex/openai/session_attachment.py +++ b/vllm_omni/entrypoints/duplex/session_attachment.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations diff --git a/vllm_omni/experimental/fullduplex/openai/session_runner.py b/vllm_omni/entrypoints/duplex/session_runner.py similarity index 99% rename from vllm_omni/experimental/fullduplex/openai/session_runner.py rename to vllm_omni/entrypoints/duplex/session_runner.py index e3a8a8cdf..821e2c9c7 100644 --- a/vllm_omni/experimental/fullduplex/openai/session_runner.py +++ b/vllm_omni/entrypoints/duplex/session_runner.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import asyncio @@ -9,34 +12,34 @@ from copy import deepcopy from fastapi import WebSocket, WebSocketDisconnect from vllm.logger import init_logger -from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.openai.audio import convert_input_audio_with_rate -from vllm_omni.experimental.fullduplex.openai.commit_policy import ( +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.entrypoints.duplex.audio import convert_input_audio_with_rate +from vllm_omni.entrypoints.duplex.commit_policy import ( CommitAction, CommitSnapshot, decide_commit_action, ) -from vllm_omni.experimental.fullduplex.openai.protocol import ( +from vllm_omni.entrypoints.duplex.protocol import ( DuplexOverlapPolicy, DuplexPlaybackCommitPolicy, DuplexSession, DuplexSessionState, DuplexTurnEventType, ) -from vllm_omni.experimental.fullduplex.openai.realtime_session import ( +from vllm_omni.entrypoints.duplex.realtime_session import ( NativeRealtimeSessionProtocol, ) -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( +from vllm_omni.entrypoints.duplex.runtime_adapter import ( PcmAppendReservation, ServingRuntimeConfigError, ServingRuntimeSessionState, payload_turn_id, ) -from vllm_omni.experimental.fullduplex.openai.session_attachment import ( +from vllm_omni.entrypoints.duplex.session_attachment import ( DuplexJournalOverflowError, ) -from vllm_omni.experimental.fullduplex.openai.websocket import ( +from vllm_omni.entrypoints.duplex.websocket import ( DuplexWebSocketActor, is_input_event, normalize_duplex_input_event, @@ -1502,6 +1505,9 @@ class DuplexSessionRunnerMixin: # the speak/listen decision and MUST receive silence units -- # the official model typically starts speaking during the # silence after a question. + # No response_id: a silence-skip listen can arrive + # while a response is still streaming and must not + # read as that response's terminal decision. await emit_event( { "type": "response.listen", @@ -1605,6 +1611,8 @@ class DuplexSessionRunnerMixin: "no_response": True, } ) + # No response_id: this silence listen answers an empty + # commit, it is not a response terminal. await emit_event( { "type": "response.listen", diff --git a/vllm_omni/experimental/fullduplex/openai/vad.py b/vllm_omni/entrypoints/duplex/vad.py similarity index 98% rename from vllm_omni/experimental/fullduplex/openai/vad.py rename to vllm_omni/entrypoints/duplex/vad.py index f81559613..362e500ee 100644 --- a/vllm_omni/experimental/fullduplex/openai/vad.py +++ b/vllm_omni/entrypoints/duplex/vad.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import binascii diff --git a/vllm_omni/experimental/fullduplex/openai/websocket.py b/vllm_omni/entrypoints/duplex/websocket.py similarity index 99% rename from vllm_omni/experimental/fullduplex/openai/websocket.py rename to vllm_omni/entrypoints/duplex/websocket.py index 3b3dc4d5d..df50fd0c3 100644 --- a/vllm_omni/experimental/fullduplex/openai/websocket.py +++ b/vllm_omni/entrypoints/duplex/websocket.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations diff --git a/vllm_omni/experimental/fullduplex/request_client.py b/vllm_omni/entrypoints/duplex_request_client.py similarity index 97% rename from vllm_omni/experimental/fullduplex/request_client.py rename to vllm_omni/entrypoints/duplex_request_client.py index d39a037a2..d9e70f672 100644 --- a/vllm_omni/experimental/fullduplex/request_client.py +++ b/vllm_omni/entrypoints/duplex_request_client.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -8,17 +8,17 @@ import time from collections.abc import Callable, MutableMapping from typing import Protocol -from vllm_omni.engine.messages import ErrorMessage, OutputMessage -from vllm_omni.entrypoints.client_request_state import ClientRequestState -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex.contracts import ( duplex_data_plane_request_info, duplex_resource_request_id, ) -from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.output import get_duplex_output_decision +from vllm_omni.engine.duplex.lease import DuplexLeaseActivity +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.messages import ErrorMessage, OutputMessage +from vllm_omni.entrypoints.client_request_state import ClientRequestState from vllm_omni.metrics.stats import OrchestratorAggregator as OrchestratorMetrics from vllm_omni.outputs import OmniRequestOutput +from vllm_omni.outputs.duplex import get_duplex_output_decision class DuplexEnginePort(Protocol): diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 80e438ee4..3d9403bf8 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -107,8 +107,9 @@ from vllm_omni.config.endpoint_policy import ( ) from vllm_omni.diffusion.models.interface import ReferenceVideoDecodeSpec from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.entrypoints.duplex.capability import should_enable_duplex_endpoint +from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler from vllm_omni.entrypoints.openai.batch_serving import OmniOpenAIServingChatBatch -from vllm_omni.entrypoints.openai.duplex_capability import should_enable_duplex_endpoint from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError from vllm_omni.entrypoints.openai.image_api_utils import ( SUPPORTED_LAYERED_RESOLUTIONS, @@ -1287,8 +1288,6 @@ async def omni_init_app_state( state.stage_configs, config_path=getattr(args, "deploy_config", None), ): - from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler - state.openai_serving_duplex = OmniDuplexSessionHandler( chat_service=state.openai_serving_chat, duplex_session_config=getattr(engine_client, "duplex_session_config", None), diff --git a/vllm_omni/experimental/fullduplex/README.md b/vllm_omni/experimental/fullduplex/README.md index e08c5ee1c..b6c918fd6 100644 --- a/vllm_omni/experimental/fullduplex/README.md +++ b/vllm_omni/experimental/fullduplex/README.md @@ -1,43 +1,41 @@ -# Experimental Full-Duplex Runtime +# Experimental Full-Duplex (JoyVL) -This package contains three experimental integrations: +This package now contains the JoyVL framework and its example integration, +plus two client-side helpers kept for the benchmark and e2e drivers: -- the existing JoyVL framework and example integration; -- the MiniCPM-o 4.5 native audio path used by `/v1/duplex` and - `/v1/realtime?duplex=1`; -- the PersonaPlex lockstep speech-to-speech path (browser demo + batched - serving, see `recipes/NVIDIA/PersonaPlex.md`). - -For the MiniCPM active runtime path, lifecycle invariants, capability boundary, -Realtime response contract, and validation scope, see [`DESIGN.md`](DESIGN.md). +```text +core/ generic duplex scaffold used by the JoyVL adapter +joyvl/ JoyVL model-specific integration +client.py legacy Realtime probe client (RealtimeDuplexClient) used by + the omniinteract / omni-duplex-eval benchmarks and the + server-VAD and Nemotron e2e drivers; applications should use + vllm_omni.clients.duplex.DuplexClient instead +video_stacking.py camera-frame tiling for omni duplex video input +``` To run JoyVL, see [`recipes/JD/JoyAI-VL-Interaction.md`](../../../recipes/JD/JoyAI-VL-Interaction.md). -## Package boundaries +The MiniCPM-o 4.5 and PersonaPlex native full-duplex runtimes graduated out +of this package. They now live in the stable tree: ```text -core/ model-agnostic duplex contracts (adapter, session, turn runtime) -engine/ AsyncOmni/orchestrator scheduler data-plane adapter -openai/ WebSocket transport, Realtime projection, and audio codecs -minicpmo45/ MiniCPM input framing, policy, compatibility, and Stage0 state -joyvl/ JoyVL model-specific integration -personaplex/ PersonaPlex lockstep engine, model-owned runtime, and serving +vllm_omni/engine/duplex/ engine control plane, sessions, leases +vllm_omni/entrypoints/duplex/ WebSocket serving and Realtime projection +vllm_omni/entrypoints/duplex_request_client.py request/output lifecycle +vllm_omni/model_executor/models/minicpmo_4_5/duplex/ MiniCPM adapter +vllm_omni/model_executor/models/personaplex/duplex/ PersonaPlex adapter +vllm_omni/model_executor/models/nemotron_voicechat/duplex/ Nemotron VoiceChat adapter +vllm_omni/model_executor/duplex_sampling.py AR-runner sampling hook helper +vllm_omni/outputs/duplex.py typed output decision envelope ``` -MiniCPM does not run through the experimental `core.DuplexRuntime` -facade. Its active path uses the `openai` session controller, the experimental -engine contracts, the standard scheduler/model runners, and an injected -MiniCPM-specific runtime extension from `minicpmo45/runtime.py`. - -`personaplex/` is a Moshi-class, pure-lockstep speech-to-speech model on the -`core/` contracts. It keeps `core/` untouched: the lockstep lifecycle (ONE -eternal, frame-clocked response that drains on close, instead of the turn-style -start/cancel-per-trigger one) is model policy and lives in the model package as -`PersonaPlexDuplexRuntime`, mirroring the model-owned runtime shape of the -MiniCPM-o duplex work. Its runnable serving path is `personaplex/serving/` -(single-session lease or `--batch-size` elastic slots) over -`personaplex/session.py` (lockstep driver). +For their architecture and validation scope, see +[`docs/design/fullduplex.md`](../../../docs/design/fullduplex.md) and +[`docs/design/fullduplex-personaplex.md`](../../../docs/design/fullduplex-personaplex.md). +PersonaPlex's single-process demo tier (browser client, standalone Moshi-web +server, `core/`-scaffold adapter) was demo-only and was removed rather than +graduated; the production path serves through the generic `/v1/duplex` stack. ## Adding a full-duplex model on the core contracts @@ -49,7 +47,10 @@ only model policy. model-specific code there and do not touch `core/`. 2. Implement one `DuplexAdapter` (`capabilities` / `on_input` / `respond`; the rest have defaults). Turn-based models run through `core.DuplexRuntime` - unchanged; a model needing a different lifecycle carries its own runtime in - its package (see `personaplex/adapter.py::PersonaPlexDuplexRuntime`). + unchanged. 3. Promote a helper from a model package up into `core/` only once a second model actually needs it. + +For production serving, prefer the stable plugin seams instead +(`duplex_serving_adapter` / `duplex_runtime_extension` dotted strings in the +model's `pipeline.py`), as MiniCPM-o 4.5 and PersonaPlex do. diff --git a/vllm_omni/experimental/fullduplex/client.py b/vllm_omni/experimental/fullduplex/client.py index 7c114f6c8..252fc92df 100644 --- a/vllm_omni/experimental/fullduplex/client.py +++ b/vllm_omni/experimental/fullduplex/client.py @@ -130,7 +130,7 @@ def build_realtime_url( if model: query["model"] = model if native_duplex is not None: - query["minicpmo45_native_duplex"] = "1" if native_duplex else "0" + query["native_duplex"] = "1" if native_duplex else "0" if autostart is not None: query["autostart"] = "1" if autostart else "0" if session_id: @@ -526,12 +526,12 @@ class RealtimeDuplexClient: session_extra_body.update( { "auto_response": auto_response, - "minicpmo45_native_duplex": True, + "native_duplex": True, "force_listen_count": 0, } ) else: - session_extra_body["minicpmo45_native_duplex"] = False + session_extra_body["native_duplex"] = False session: dict[str, object] = { "model": model, "modalities": ["audio", "text"], diff --git a/vllm_omni/experimental/fullduplex/engine/__init__.py b/vllm_omni/experimental/fullduplex/engine/__init__.py deleted file mode 100644 index c8e20437f..000000000 --- a/vllm_omni/experimental/fullduplex/engine/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""Engine-side helpers for the experimental full-duplex runtime.""" diff --git a/vllm_omni/experimental/fullduplex/personaplex/__init__.py b/vllm_omni/experimental/fullduplex/personaplex/__init__.py deleted file mode 100644 index bf6c711bb..000000000 --- a/vllm_omni/experimental/fullduplex/personaplex/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""PersonaPlex full-duplex backend for the experimental fullduplex framework. - -PersonaPlex (``nvidia/personaplex-7b-v1``) is a Moshi finetune: a pure-lockstep -speech-to-speech model. This package plugs it into the model-agnostic -``fullduplex/core`` seam without touching the engine: - -- :class:`PersonaPlexConfig` immutable session config (voice / persona / sampling) -- :class:`PersonaPlexEngine` the moshi-free lockstep engine (``FrameStepper``; - single session and B concurrent slots with elastic per-slot recycle) -- :class:`PersonaPlexSession` lockstep driver (the runnable serving primitive) -- :class:`PersonaPlexDuplexAdapter` the ``core.DuplexAdapter`` for lockstep audio -- :class:`PersonaPlexDuplexRuntime` the model-owned lockstep lifecycle - (start-once / drain-on-close; ``core`` stays verbatim upstream) -""" - -from vllm_omni.experimental.fullduplex.personaplex.adapter import ( - PersonaPlexDuplexAdapter, - PersonaPlexDuplexRuntime, -) -from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig -from vllm_omni.experimental.fullduplex.personaplex.engine import ( - FrameOutput, - FrameStepper, - PersonaPlexEngine, -) -from vllm_omni.experimental.fullduplex.personaplex.runtime import PrefillStep -from vllm_omni.experimental.fullduplex.personaplex.session import PersonaPlexSession - -__all__ = [ - "FrameOutput", - "FrameStepper", - "PersonaPlexConfig", - "PersonaPlexDuplexAdapter", - "PersonaPlexDuplexRuntime", - "PersonaPlexEngine", - "PersonaPlexSession", - "PrefillStep", -] diff --git a/vllm_omni/experimental/fullduplex/personaplex/adapter.py b/vllm_omni/experimental/fullduplex/personaplex/adapter.py deleted file mode 100644 index 1508931f8..000000000 --- a/vllm_omni/experimental/fullduplex/personaplex/adapter.py +++ /dev/null @@ -1,177 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""PersonaPlex duplex adapter + model-owned lockstep runtime. - -PersonaPlex is a *continuous* (frame-clocked, no turn-taking) model: one eternal -``respond()`` that pulls one 80 ms user frame at a time and emits one agent -frame + text token. The shared ``core.DuplexRuntime`` implements the turn -lifecycle (start/cancel per trigger), which does not fit lockstep, so the -lifecycle lives HERE as :class:`PersonaPlexDuplexRuntime` — model policy stays -in the model package, mirroring #3907's model-owned minicpmo45 runtime, and -``core`` remains verbatim upstream. - -Stepping happens off the event loop (``asyncio.to_thread``, as Moshi's own -server does) so the input side stays live — that is what makes barge-in work -without any explicit flush. - -Depends only on the ``FrameStepper`` protocol, so it is exercised by GPU-free -tests with a stub stepper; ``PersonaPlexEngine`` is the real backend. -""" - -from __future__ import annotations - -import asyncio -from collections.abc import AsyncIterator -from typing import Any - -import numpy as np - -from vllm_omni.experimental.fullduplex.core import protocol as ev -from vllm_omni.experimental.fullduplex.core.adapter import DuplexAdapter, DuplexCapability, OutputChunk -from vllm_omni.experimental.fullduplex.core.runtime import DuplexRuntime, Emit -from vllm_omni.experimental.fullduplex.core.session import DuplexSession, DuplexState -from vllm_omni.experimental.fullduplex.personaplex.engine import FrameStepper -from vllm_omni.experimental.fullduplex.personaplex.session import MAX_PENDING_INPUT, offer_realtime - -_CLOSE = object() # sentinel: drain the inbox and end the eternal response - - -class PersonaPlexDuplexAdapter(DuplexAdapter): - """Drive a :class:`FrameStepper` as a continuous full-duplex response.""" - - def __init__( - self, - stepper: FrameStepper, - *, - voice_prompt: str | None = None, - persona: str | None = None, - ) -> None: - self._stepper = stepper - self._voice_prompt = voice_prompt - self._persona = persona - self._frame_size = stepper.frame_size - # Bounded, drop-oldest: input arriving faster than the stepper can run - # must shed stale frames (realtime), not grow memory + latency unbounded. - self._inbox: asyncio.Queue[Any] = asyncio.Queue(maxsize=MAX_PENDING_INPUT) - self._pending = np.zeros(0, dtype=np.float32) - self._opened = False - - def capabilities(self) -> DuplexCapability: - return DuplexCapability( - input_modalities=frozenset({"audio"}), - output_modalities=frozenset({"audio", "text"}), - proactive=True, - ) - - async def on_input(self, session: DuplexSession, modality: str, data: Any) -> None: - """Frame incoming PCM into exact 80 ms windows and queue them for the loop.""" - if modality != "audio": - return - samples = np.ascontiguousarray(data, dtype=np.float32).reshape(-1) - self._pending = np.concatenate([self._pending, samples]) if self._pending.size else samples - n = self._frame_size - while self._pending.shape[0] >= n: - frame, self._pending = self._pending[:n], self._pending[n:] - offer_realtime(self._inbox, frame) - - async def respond(self, session: DuplexSession) -> AsyncIterator[OutputChunk]: - """The eternal lockstep loop: one user frame in, one agent frame + text out.""" - if not self._opened: - await asyncio.to_thread(self._stepper.open_session, self._voice_prompt, self._persona) - self._opened = True - while True: - frame = await self._inbox.get() - if frame is _CLOSE: - break - out = await asyncio.to_thread(self._stepper.step, frame) - if out.audio is not None: - yield OutputChunk("audio", out.audio) - if out.text: - yield OutputChunk("text", out.text) - - async def on_close(self, session: DuplexSession) -> None: - # Model-owned drain signal (not part of the core DuplexAdapter ABC): - # PersonaPlexDuplexRuntime calls this on every exit path. Pad + flush a - # partial final frame (same contract as PersonaPlexSession.flush()) so - # the tail of the user's audio is stepped, not dropped. - if self._pending.size: - pad = self._frame_size - self._pending.shape[0] - frame = np.concatenate([self._pending, np.zeros(pad, dtype=np.float32)]) - self._pending = np.zeros(0, dtype=np.float32) - offer_realtime(self._inbox, frame) - offer_realtime(self._inbox, _CLOSE) - - async def on_barge_in(self, session: DuplexSession) -> None: - # No-op: Moshi-class models perceive the user continuously, so an interrupt - # needs no downstream flush — the next frames simply steer the model. - return - - -class PersonaPlexDuplexRuntime(DuplexRuntime): - """Model-owned lockstep lifecycle: start once, never supersede, drain on exit. - - ``core.DuplexRuntime`` runs the turn lifecycle — every trigger starts a new - response that supersedes and cancels the previous one. A lockstep model has - exactly ONE response for the whole session: it consumes user frames as they - arrive and only ends when the input side goes away. Keeping this here (not - as a flag on the shared runtime) mirrors #3907's model-owned minicpmo45 - runtime: model policy lives in the model package. - """ - - async def run(self, inputs: AsyncIterator[dict], emit: Emit) -> None: - task: asyncio.Task | None = None - try: - async for event in inputs: - etype = event.get("type") - if etype == ev.INPUT_APPEND: - modality = event.get("modality", "") - if modality not in self._capabilities.input_modalities: - await emit(ev.error(f"unsupported input modality: {modality}")) - continue - await self.adapter.on_input(self.session, modality, event.get("data")) - self.session.state = DuplexState.LISTENING - task = self._ensure_started(task, emit) - elif etype in (ev.INPUT_COMMIT, ev.RESPONSE_CREATE): - task = self._ensure_started(task, emit) - elif etype == ev.RESPONSE_CANCEL: - # Barge-in is native for lockstep models: bump the epoch for the - # playback cursor but keep the single response streaming. - await self._barge_in() - await emit(ev.cancelled(self.session.response_index)) - elif etype == ev.PLAYBACK_ACK: - await self.adapter.on_playback_ack(self.session, int(event.get("cursor", 0))) - elif etype == ev.CLOSE: - break - finally: - # Drain on EVERY exit path: CLOSE, iterator EOF, transport errors, - # and cancellation must not leave the eternal response blocked on - # its inbox or the session outside CLOSED. - try: - if task is not None and not task.done(): - await self.adapter.on_close(self.session) - await task - finally: - self.session.state = DuplexState.CLOSED - - def _ensure_started(self, task: asyncio.Task | None, emit: Emit) -> asyncio.Task: - # Start the single eternal response exactly once; never supersede it. - if task is None or task.done(): - return asyncio.create_task(self._respond_once(emit)) - return task - - async def _respond_once(self, emit: Emit) -> None: - # One response for the whole session. No staleness checks: the loop ends - # only when the adapter's respond() returns (after on_close drains its - # input). - response_index, _ = self.session.begin_response() - await emit(ev.created(response_index)) - try: - async for chunk in self.adapter.respond(self.session): - await emit(ev.delta(response_index, chunk.modality, chunk.data)) - except asyncio.CancelledError: - raise - except Exception as err: - await emit(ev.error(f"response failed: {err}")) - finally: - await emit(ev.done(response_index)) - self.session.end_response() diff --git a/vllm_omni/experimental/fullduplex/personaplex/engine.py b/vllm_omni/experimental/fullduplex/personaplex/engine.py deleted file mode 100644 index 488676aa1..000000000 --- a/vllm_omni/experimental/fullduplex/personaplex/engine.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""PersonaPlex duplex protocol home (moshi-free). - -Defines the narrow ``FrameStepper`` seam (one 80 ms user frame in, one agent -frame + text piece out) that the session driver, the duplex adapter and the -GPU-free tests depend on, and re-exports the native lockstep engine that -implements it. The implementation lives in ``runtime.py`` and is built -entirely from vllm-omni-native components (streaming Helium temporal, input -embeddings, depformer, streaming Mimi codec) — the external ``moshi`` package -is no longer used. -""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass -from typing import TYPE_CHECKING, Protocol, runtime_checkable - -import numpy as np - -if TYPE_CHECKING: - from numpy.typing import NDArray - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class FrameOutput: - """Result of stepping the model by exactly one 80 ms frame. - - Attributes: - audio: Agent PCM for this frame (``frame_size`` mono float32 samples), or - ``None`` during the model's initial delay/warmup frames. - text: Inner-monologue text piece for this frame, or ``None`` when the - model emitted a padding token (silence between words). - """ - - audio: NDArray[np.float32] | None - text: str | None - - -@runtime_checkable -class FrameStepper(Protocol): - """The single seam between PersonaPlex and the duplex framework. - - A stepper consumes one user audio frame and returns one agent frame. It owns - all conversation state (KV caches / streaming buffers); callers never re-feed - history. ``PersonaPlexEngine`` is the real implementation; tests pass a stub. - """ - - sample_rate: int - frame_size: int - - def open_session(self, voice_prompt: str | None = None, persona: str | None = None) -> None: - """Reset streaming state and inject the voice clone + persona prompt.""" - ... - - def step(self, user_pcm: NDArray[np.float32]) -> FrameOutput: - """Advance the conversation by one frame of exactly ``frame_size`` samples.""" - ... - - -from vllm_omni.experimental.fullduplex.personaplex.runtime import ( # noqa: E402 - PersonaPlexEngine, - PrefillStep, -) - -__all__ = [ - "FrameOutput", - "FrameStepper", - "PersonaPlexEngine", - "PrefillStep", -] diff --git a/vllm_omni/experimental/fullduplex/personaplex/runtime.py b/vllm_omni/experimental/fullduplex/personaplex/runtime.py deleted file mode 100644 index 885b36aa5..000000000 --- a/vllm_omni/experimental/fullduplex/personaplex/runtime.py +++ /dev/null @@ -1,473 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Moshi-free PersonaPlex lockstep engine (native temporal + depformer + Mimi). - -This is the real-time duplex backend built entirely from vllm-omni-native -components: :class:`PersonaPlexTemporalStreaming` (ring-KV Helium), -:class:`PersonaPlexInputEmbeddings`, :class:`PersonaPlexDepformer` and -:class:`PersonaPlexMimiCodec`. It reimplements the reference lockstep semantics -(staging ring, per-codebook delays, warmup gates, hybrid system-prompt prefill, -per-slot elastic recycle) as first-class policy code, so the external ``moshi`` -package is no longer needed at runtime. - -Lockstep contract per 80 ms tick (17 token rows: text, 8 agent codebooks, 8 -user codebooks; delays ``[0, 1x8-ish from checkpoint]``): - -1. Mimi-encode the user frame and write it (delayed) into the staging ring. -2. Read the model input at ``(o - 1) % CT``, embed, step the temporal. -3. Greedy text token; depformer expands the 16 audio rows (teacher-forced where - the staging ring already holds provided values). -4. Write sampled rows at ``o % CT``; gather the de-delayed output frame; decode - agent rows 1..8 through Mimi (rows still in warmup are sanitized to silence - tokens: the LM's initial audio token id equals the codec cardinality and is - out of range for the codebook tables). - -Elastic recycle mirrors the verified design from the moshi-hosted path: the -temporal ring KV masks the recycled row (plus the one LM "sacrifice tick" -write), the staging row is wiped to initial tokens, and the system prompt -(voice embeddings, silence spacer, persona tokens, silence) is replayed per-row -while other slots keep conversing. -""" - -from __future__ import annotations - -import logging -import os -import tarfile -import threading -from pathlib import Path -from typing import TYPE_CHECKING - -import numpy as np - -from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig -from vllm_omni.experimental.fullduplex.personaplex.policy import ( - AUDIO_INITIAL_TOKEN, - AUDIO_SILENCE_FRAME_CNT, - NON_TEXT_TOKEN_IDS, - SILENCE_TOKENS, - SINE_TOKENS, - TEXT_INITIAL_TOKEN, - TEXT_VOCAB_SIZE, - ZERO_TEXT_TOKEN, - PrefillStep, - wrap_with_system_tags, -) - -if TYPE_CHECKING: - from numpy.typing import NDArray - -logger = logging.getLogger(__name__) - - -class PersonaPlexEngine: - """Frame-lockstep PersonaPlex on native components only (no moshi). - - One class serves both surfaces: the single-session ``FrameStepper`` protocol - (``open_session`` / ``step``) and the batched slot API consumed by - ``serving/batched.py`` (``open_batch`` / ``prefill_steps`` / ``recycle_slot`` - / ``voice_end`` / ``end_prefill`` / ``step_batch``). - """ - - def __init__(self, config: PersonaPlexConfig | None = None) -> None: - self.config = config or PersonaPlexConfig() - self.sample_rate = self.config.sample_rate - self.frame_size = self.config.frame_size - # Serializes the single-session FrameStepper surface (open_session/step) - # across worker threads: a cancelled asyncio task cannot stop a running - # engine call, so a racing open() must wait for the in-flight step. - # The batched surface is serialized by BatchedSessionManager instead. - self._fs_lock = threading.Lock() - self._loaded = False - self._opened = False - self._tokenizer = None - self._codec = None - self._temporal = None - self._emb = None - self._dep = None - self._voice_embeddings = None # [T, 1, 1, H] - self._voice_cache = None # [1, K, CT] - self._loaded_voice = None # name/path of the currently loaded voice - # lockstep state (built at open time) - self._cache = None # [B, K, CT] long - self._provided = None # [B, K, CT] bool - self._initial = None # [K] long - self._delays = None # [K] long (python list) - self._phys = 0 - self._local = None # [B] cpu long - - # -- loading -------------------------------------------------------------- - - def load(self) -> PersonaPlexEngine: - if self._loaded: - return self - import sentencepiece - import torch - from huggingface_hub import hf_hub_download - from safetensors.torch import load_file - - from vllm_omni.model_executor.models.personaplex.configuration_personaplex import ( - PersonaPlexConfig as _PPModelConfig, - ) - from vllm_omni.model_executor.models.personaplex.personaplex_depformer import ( - PersonaPlexDepformer, - ) - from vllm_omni.model_executor.models.personaplex.personaplex_embeddings import ( - PersonaPlexInputEmbeddings, - ) - from vllm_omni.model_executor.models.personaplex.personaplex_mimi import ( - PersonaPlexMimiCodec, - ) - from vllm_omni.model_executor.models.personaplex.personaplex_temporal import ( - PersonaPlexTemporalStreaming, - ) - - cfg = self.config - B = cfg.batch_size - logger.info("PersonaPlex native: loading Mimi codec") - self._codec = PersonaPlexMimiCodec(device=cfg.device) - self._codec.streaming_init(B) - - tok_path = hf_hub_download(cfg.hf_repo, "tokenizer_spm_32k_3.model") - self._tokenizer = sentencepiece.SentencePieceProcessor(tok_path) - - logger.info("PersonaPlex native: loading temporal + depformer + embeddings") - sd = load_file(hf_hub_download(cfg.hf_repo, "model.safetensors"), device=cfg.device) - dtype = torch.bfloat16 - self._temporal = PersonaPlexTemporalStreaming().to(cfg.device, dtype) - self._temporal.load_weights(sd) - self._temporal.eval() - self._temporal.streaming_init(B) - - ppcfg = _PPModelConfig() - self._emb = PersonaPlexInputEmbeddings(ppcfg).to(cfg.device, dtype) - self._emb.load_weights(sd) - self._emb.eval() - self._dep = PersonaPlexDepformer(ppcfg.depformer_config, temporal_hidden_size=4096, text_card=32000).to( - cfg.device, dtype - ) - self._dep.load_weights(sd) - self._dep.eval() - del sd - torch.accelerator.empty_cache() - - K = 17 - self._delays = [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1] # text, agent cb0..7, user cb0..7 - self._initial = torch.tensor( - [TEXT_INITIAL_TOKEN] + [AUDIO_INITIAL_TOKEN] * 16, dtype=torch.long, device=cfg.device - ) - ct = max(self._delays) + 3 - self._cache = torch.full((B, K, ct), AUDIO_INITIAL_TOKEN, dtype=torch.long, device=cfg.device) - self._provided = torch.zeros((B, K, ct), dtype=torch.bool, device=cfg.device) - self._local = torch.zeros(B, dtype=torch.long) - self._sine = torch.tensor(SINE_TOKENS, dtype=torch.long, device=cfg.device) - self._silence = torch.tensor(SILENCE_TOKENS, dtype=torch.long, device=cfg.device) - - self._load_voice(cfg.voice_prompt) - self._loaded = True - return self - - def _load_voice(self, voice: str) -> None: - import torch - - if voice == self._loaded_voice: - return - path = self._resolve_voice_prompt(voice) - if not path.endswith(".pt"): - raise ValueError(f"native stack needs a .pt voice-embedding bundle; got {path!r}") - state = torch.load(path, map_location=self.config.device, weights_only=True) - self._voice_embeddings = state["embeddings"].to(self.config.device) - self._voice_cache = state["cache"].to(self.config.device) - self._loaded_voice = voice - - def _resolve_voice_prompt(self, voice: str) -> str: - from huggingface_hub import hf_hub_download - - if os.path.exists(voice): - return voice - tgz = Path(hf_hub_download(self.config.hf_repo, "voices.tgz")) - vdir = tgz.parent / "voices" - if not vdir.exists(): - with tarfile.open(tgz, "r:gz") as tar: - tar.extractall(path=tgz.parent) # noqa: S202 - trusted first-party bundle - cand = vdir / voice - if not cand.exists(): - raise FileNotFoundError(f"voice prompt {voice!r} not found in voices.tgz") - return str(cand) - - # -- lockstep core ---------------------------------------------------------- - - def _reset_all_streaming(self) -> None: - self._codec.reset_streaming() - self._temporal.reset_streaming() - self._cache.fill_(AUDIO_INITIAL_TOKEN) - self._cache[:, 0, :] = TEXT_INITIAL_TOKEN - self._provided.fill_(False) - self._phys = 0 - self._local.zero_() - - def _tick( - self, - user_codes, # [B, 8] long or None (prefill rows overridden separately) - forced_moshi=None, # [B, 8] long (values used on force_mask rows) - forced_text=None, # [B] long - force_mask=None, # [B] bool (device) - embed_override=None, # ([B, 1, H], [B] bool rows) - ): - """One lockstep step. Returns (ran, out[B,17] | None).""" - import torch - - B, K, CT = self._cache.shape - o = self._phys - lo = self._local - - if user_codes is not None: - for q in range(8): - k = 9 + q - wp = (o + self._delays[k]) % CT - self._cache[:, k, wp] = user_codes[:, q] - self._provided[:, k, wp] = True - if forced_moshi is not None and force_mask is not None: - for q in range(8): - k = 1 + q - wp = (o + self._delays[k]) % CT - self._cache[force_mask, k, wp] = forced_moshi[force_mask, q] - self._provided[force_mask, k, wp] = True - if forced_text is not None and force_mask is not None: - wp = (o + self._delays[0]) % CT - self._cache[force_mask, 0, wp] = forced_text[force_mask] - self._provided[force_mask, 0, wp] = True - - for k, delay in enumerate(self._delays): - warm = lo <= delay # [B] cpu bool - if bool(warm.any()): - wd = warm.to(self._cache.device) - pos = o % CT - self._cache[:, k, pos] = torch.where(wd, self._initial[k].expand(B), self._cache[:, k, pos]) - self._provided[:, k, pos] = self._provided[:, k, pos] | wd - - if o == 0: - # Genuine cold start: no prior physical frame to read as model input. - self._cache[:, :, 0] = self._initial.view(1, K).expand(B, K) - self._phys += 1 - self._local += 1 - return False, None - - mip = (o - 1) % CT - tp = o % CT - input_ = self._cache[:, :, mip] # [B, K] - target_ = self._cache[:, :, tp] - provided_ = self._provided[:, :, tp] - - emb = self._emb(input_.unsqueeze(-1)) # [B, 1, H] - if embed_override is not None: - ov, rows = embed_override - emb = torch.where(rows.view(B, 1, 1), ov.to(emb.dtype), emb) - hidden, text_logits = self._temporal.step(emb) - - sampled_text = text_logits.float().view(B, -1).argmax(-1) # greedy - next_text = torch.where(provided_[:, 0], target_[:, 0], sampled_text) - audio = self._dep(next_text, hidden, audio_tokens=target_[:, 1:], audio_provided=provided_[:, 1:]) # [B, 16] - - self._provided[:, :, mip] = False - self._cache[:, 0, tp] = torch.where(~provided_[:, 0], sampled_text, self._cache[:, 0, tp]) - self._cache[:, 1:, tp] = torch.where(~provided_[:, 1:], audio, self._cache[:, 1:, tp]) - - max_delay = max(self._delays) - valid = lo > max_delay - if not bool(valid.any()): - self._phys += 1 - self._local += 1 - return True, None - - idx = torch.tensor( - [(self._phys - max_delay + d) % CT for d in self._delays], - device=self._cache.device, - dtype=torch.long, - ) - out = self._cache.gather(2, idx.view(1, K, 1).expand(B, K, 1)).squeeze(-1) # [B, K] - self._last_valid = valid.clone() - self._phys += 1 - self._local += 1 - return True, out - - def _boot_prefill(self, persona: str | None) -> None: - """Batch-wide hybrid system prompt: voice -> cache restore -> silence -> - persona -> silence (all rows identical).""" - import torch - - B = self.config.batch_size - dev = self.config.device - dummy_user = self._initial[9:].view(1, 8).expand(B, 8) - dummy_moshi = self._initial[1:9].view(1, 8).expand(B, 8) - zero_text = torch.full((B,), ZERO_TEXT_TOKEN, dtype=torch.long, device=dev) - all_rows = torch.ones(B, dtype=torch.bool, device=dev) - - for e in self._voice_embeddings: - ov = e.view(1, 1, -1).expand(B, 1, -1) - while True: - ran, _ = self._tick(dummy_user, dummy_moshi, zero_text, all_rows, embed_override=(ov, all_rows)) - if ran: - break - self._cache.copy_(self._voice_cache.expand_as(self._cache)) - - sine = self._sine.view(1, 8).expand(B, 8) - silence = self._silence.view(1, 8).expand(B, 8) - text = persona if persona is not None else self.config.persona - toks = self._tokenizer.encode(wrap_with_system_tags(text)) if text else [] - for phase_toks in ( - [ZERO_TEXT_TOKEN] * AUDIO_SILENCE_FRAME_CNT, - toks, - [ZERO_TEXT_TOKEN] * AUDIO_SILENCE_FRAME_CNT, - ): - for t in phase_toks: - tt = torch.full((B,), int(t), dtype=torch.long, device=dev) - self._tick(sine, silence, tt, all_rows) - self._codec.reset_streaming() - - # -- batched surface (mirrors the previous engine's API) -------------------- - - def open_batch(self) -> None: - if not self._loaded: - self.load() - self._reset_all_streaming() - self._boot_prefill(None) - self._opened = True - logger.info("PersonaPlex native: %d slots prefilled", self.config.batch_size) - - def prefill_steps(self, persona: str | None = None) -> list[PrefillStep]: - zero = self._silence - silence = [ - PrefillStep("tokens", moshi_tokens=zero, text_token=ZERO_TEXT_TOKEN, user_sine=True) - for _ in range(AUDIO_SILENCE_FRAME_CNT) - ] - steps: list[PrefillStep] = [PrefillStep("sacrifice")] - for e in self._voice_embeddings: - steps.append(PrefillStep("voice", embedding=e.view(1, 1, -1))) - steps.append(PrefillStep("voice_end")) - steps.extend(silence) - text = persona if persona is not None else self.config.persona - if text: - for tok in self._tokenizer.encode(wrap_with_system_tags(text)): - steps.append(PrefillStep("tokens", moshi_tokens=zero, text_token=int(tok), user_sine=True)) - steps.extend(silence) - return steps - - def recycle_slot(self, b: int) -> None: - self._temporal.reset_slot(b) - self._temporal.bump_slot_start(b) # LM sacrifice-tick semantics - self._local[b] = 0 - self._cache[b] = self._initial.view(-1, 1).expand(-1, self._cache.shape[2]) - self._provided[b] = False - - def voice_end(self, b: int) -> None: - import torch - - ct = self._cache.shape[2] - shift = int((self._phys - int(self._local[b])) % ct) - self._cache[b] = torch.roll(self._voice_cache[0], shifts=shift, dims=-1) - - def end_prefill(self, b: int) -> None: - self._codec.reset_slot(b) - - def step_batch(self, user_pcm: NDArray[np.float32], prefill: dict[int, PrefillStep] | None = None): - import torch - - if not self._opened: - raise RuntimeError("open_batch() must be called before step_batch()") - cfg = self.config - B = cfg.batch_size - pcm = np.ascontiguousarray(user_pcm, dtype=np.float32).reshape(B, self.frame_size) - codes = self._codec.encode_frame(torch.from_numpy(pcm).to(cfg.device)).long() - return self._step_from_codes(codes, prefill) - - def step_codes(self, user_codes, prefill: dict[int, PrefillStep] | None = None): - """Drive the lockstep loop from precomputed user codes (golden replays). - - Kept public deliberately: bit-exact golden-replay verification must - bypass the Mimi encoder, whose streaming conv state makes encoded - codes run-order dependent (valid but not reproducible token-for-token). - """ - return self._step_from_codes(user_codes.to(self.config.device).long(), prefill) - - def _step_from_codes(self, codes, prefill): - import torch - - from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput - - cfg = self.config - B = cfg.batch_size - dev = cfg.device - forced_moshi = forced_text = force_mask = embed_override = None - if prefill: - force_mask = torch.zeros(B, dtype=torch.bool, device=dev) - embed_rows = torch.zeros(B, dtype=torch.bool, device=dev) - emb_full = None - forced_moshi = self._initial[1:9].view(1, 8).expand(B, 8).clone() - forced_text = torch.full((B,), ZERO_TEXT_TOKEN, dtype=torch.long, device=dev) - for b, stp in prefill.items(): - if stp.kind == "voice_end": - raise ValueError("voice_end is not a tick; call engine.voice_end(b)") - codes[b] = self._sine if stp.user_sine else self._initial[9:] - if stp.kind == "voice": - embed_rows[b] = True - if emb_full is None: - emb_full = torch.zeros(B, 1, stp.embedding.shape[-1], dtype=stp.embedding.dtype, device=dev) - emb_full[b] = stp.embedding[0] - force_mask[b] = True # dummy staging like step_embeddings - else: - force_mask[b] = True - if stp.kind == "tokens": - forced_moshi[b] = stp.moshi_tokens - forced_text[b] = int(stp.text_token) - if bool(embed_rows.any()): - embed_override = (emb_full, embed_rows) - - ran, out = self._tick(codes, forced_moshi, forced_text, force_mask, embed_override) - self.last_out_tokens = None if out is None else out.detach().cpu() - if not ran or out is None: - return [None] * B - - valid = self._last_valid # [B] cpu bool - emit = [bool(valid[b]) and (not prefill or b not in prefill) for b in range(B)] - audio_tokens = out[:, 1:9].clone() - if not all(emit): - hide = torch.tensor([not e for e in emit], device=dev).view(B, 1) - audio_tokens = torch.where(hide, self._silence.view(1, 8).expand(B, 8), audio_tokens) - pcm_out = self._codec.decode_frame(audio_tokens) # [B, frame] - outs = [] - for b in range(B): - if not emit[b]: - outs.append(None) - continue - audio = pcm_out[b].float().detach().cpu().numpy().astype(np.float32) - outs.append(FrameOutput(audio=audio, text=self._decode_text(int(out[b, 0].item())))) - return outs - - # -- single-session FrameStepper surface ------------------------------------- - - def open_session(self, voice_prompt: str | None = None, persona: str | None = None) -> None: - with self._fs_lock: - if not self._loaded: - self.load() - # Always resolve a voice: falling back to the configured default here - # keeps a previous session's custom voice from leaking into this one. - self._load_voice(voice_prompt or self.config.voice_prompt) - self._reset_all_streaming() - self._boot_prefill(persona) - self._opened = True - - def step(self, user_pcm: NDArray[np.float32]): - with self._fs_lock: - pcm = np.ascontiguousarray(user_pcm, dtype=np.float32).reshape(-1) - if pcm.shape[0] != self.frame_size: - raise ValueError(f"expected {self.frame_size} samples per frame, got {pcm.shape[0]}") - outs = self.step_batch(pcm.reshape(1, -1)) - from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput - - return outs[0] if outs[0] is not None else FrameOutput(audio=None, text=None) - - def _decode_text(self, token_id: int) -> str | None: - if token_id in NON_TEXT_TOKEN_IDS or token_id >= TEXT_VOCAB_SIZE: - return None - return self._tokenizer.id_to_piece(token_id).replace("▁", " ") diff --git a/vllm_omni/experimental/fullduplex/personaplex/serving/__init__.py b/vllm_omni/experimental/fullduplex/personaplex/serving/__init__.py deleted file mode 100644 index c3044a80d..000000000 --- a/vllm_omni/experimental/fullduplex/personaplex/serving/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""PersonaPlex full-duplex WebSocket serving (PCM in / PCM + text out).""" - -from typing import Any - -__all__ = ["create_app", "main"] - - -def __getattr__(name: str) -> Any: - if name not in __all__: - raise AttributeError(name) - from vllm_omni.experimental.fullduplex.personaplex.serving import server - - return getattr(server, name) diff --git a/vllm_omni/experimental/fullduplex/personaplex/serving/batched.py b/vllm_omni/experimental/fullduplex/personaplex/serving/batched.py deleted file mode 100644 index 26abd92a8..000000000 --- a/vllm_omni/experimental/fullduplex/personaplex/serving/batched.py +++ /dev/null @@ -1,184 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Slot manager + lockstep tick loop for batched PersonaPlex serving. - -Pure-python orchestration over a :class:`PersonaPlexEngine`-shaped object -(injected, so tests run GPU-free with a stub): a fixed table of B conversation -slots, one shared wall-clock 80 ms tick that advances every non-idle slot, and -per-slot recycle prefill for new callers. - -Threading model: ``tick()`` runs on a dedicated stepping thread (the engine call -is synchronous CUDA work); ``acquire``/``release``/``feed`` are called from the -asyncio thread. All slot state is guarded by one lock; per-frame output is -handed off via each slot's ``on_output`` callback (called on the stepping -thread -- the server bridges to asyncio with ``call_soon_threadsafe``). -""" - -from __future__ import annotations - -import enum -import logging -import threading -import time -from collections import deque -from collections.abc import Callable -from dataclasses import dataclass, field -from typing import Any - -import numpy as np - -logger = logging.getLogger(__name__) - - -class SlotPhase(enum.Enum): - IDLE = "idle" - PREFILL = "prefill" - LIVE = "live" - - -@dataclass -class Slot: - index: int - phase: SlotPhase = SlotPhase.IDLE - used: bool = False # a used slot must be recycled before reuse - needs_recycle: bool = False # engine.reset executed by the tick thread only - epoch: int = 0 - pending: np.ndarray = field(default_factory=lambda: np.zeros(0, dtype=np.float32)) - inq: deque = field(default_factory=deque) - prefill: deque = field(default_factory=deque) - on_output: Callable[[Any], None] | None = None # FrameOutput, stepping thread - on_ready: Callable[[], None] | None = None # prefill finished, stepping thread - - -class BatchedSessionManager: - """Own the slot table and drive one engine tick per call to :meth:`tick`.""" - - def __init__(self, engine, max_buffered_frames: int = 25) -> None: - self.engine = engine - self.frame_size = engine.frame_size - self.batch_size = engine.config.batch_size - self.max_buffered_frames = max_buffered_frames - self.slots = [Slot(index=i) for i in range(self.batch_size)] - self.lock = threading.Lock() - # step_batch advances EVERY row each tick (lockstep), so once any tick has - # run, "pristine" idle rows have silently drifted from their boot prefill - # and must be recycled before a client may join them. - self._ticked = False - - # -- connection side (asyncio thread) ------------------------------------ - - def acquire(self, persona: str | None = None) -> tuple[Slot, bool] | None: - """Claim an idle slot. Returns ``(slot, ready_now)`` or ``None`` if full. - - A pristine slot (never used since boot, and claimed before the first - engine tick) still carries the default voice + persona from - ``open_batch`` and is live immediately when no custom persona is - requested. Otherwise the slot is recycled and its system prompt - replayed over the next ticks (``ready_now == False``) -- including - pristine rows after ticking has started, because lockstep stepping - advances every row whether or not a client is attached. - """ - with self.lock: - slot = next((s for s in self.slots if s.phase is SlotPhase.IDLE), None) - if slot is None: - return None - slot.epoch += 1 - slot.pending = np.zeros(0, dtype=np.float32) - slot.inq.clear() - slot.prefill.clear() - if not slot.used and persona is None and not self._ticked: - slot.phase = SlotPhase.LIVE - slot.used = True - return slot, True - # The engine reset itself must run BETWEEN engine steps, i.e. on the - # stepping thread -- only mark it here; tick() executes it. - slot.prefill.extend(self.engine.prefill_steps(persona)) - slot.needs_recycle = True - slot.phase = SlotPhase.PREFILL - slot.used = True - return slot, False - - def release(self, slot: Slot) -> None: - with self.lock: - slot.phase = SlotPhase.IDLE - slot.epoch += 1 - slot.on_output = None - slot.on_ready = None - slot.inq.clear() - slot.prefill.clear() - slot.pending = np.zeros(0, dtype=np.float32) - - def feed(self, slot: Slot, pcm: np.ndarray, epoch: int) -> None: - """Buffer incoming PCM and carve it into exact engine frames.""" - with self.lock: - if slot.epoch != epoch or slot.phase is SlotPhase.IDLE: - return - samples = np.ascontiguousarray(pcm, dtype=np.float32).reshape(-1) - slot.pending = np.concatenate([slot.pending, samples]) if slot.pending.size else samples - n = self.frame_size - while slot.pending.shape[0] >= n: - frame, slot.pending = slot.pending[:n], slot.pending[n:] - slot.inq.append(frame) - while len(slot.inq) > self.max_buffered_frames: - slot.inq.popleft() # realtime: drop the oldest backlog - - # -- stepping side (dedicated thread) ------------------------------------ - - def tick(self) -> bool: - """Advance every non-idle slot one frame. Returns False when all idle.""" - with self.lock: - if all(s.phase is SlotPhase.IDLE for s in self.slots): - return False - self._ticked = True - pcm = np.zeros((self.batch_size, self.frame_size), dtype=np.float32) - prefill: dict[int, Any] = {} - emitters: list[tuple[Slot, int]] = [] - for slot in self.slots: - if slot.phase is SlotPhase.LIVE: - if slot.inq: - pcm[slot.index] = slot.inq.popleft() - emitters.append((slot, slot.epoch)) - elif slot.phase is SlotPhase.PREFILL: - if slot.needs_recycle: - self.engine.recycle_slot(slot.index) - slot.needs_recycle = False - while slot.prefill and slot.prefill[0].kind == "voice_end": - self.engine.voice_end(slot.index) - slot.prefill.popleft() - if slot.prefill: - prefill[slot.index] = slot.prefill.popleft() - else: - self.engine.end_prefill(slot.index) - slot.phase = SlotPhase.LIVE - emitters.append((slot, slot.epoch)) - if slot.on_ready is not None: - slot.on_ready() - - outs = self.engine.step_batch(pcm, prefill or None) - - with self.lock: - for slot, epoch in emitters: - out = outs[slot.index] - if out is None or slot.epoch != epoch or slot.on_output is None: - continue - slot.on_output(out) - return True - - def run_forever(self, stop: threading.Event, frame_seconds: float = 0.08) -> None: - """Wall-clock-paced stepping loop (run on a dedicated thread).""" - next_t = time.monotonic() - while not stop.is_set(): - did = self.tick() - if not did: - time.sleep(frame_seconds / 4) - next_t = time.monotonic() - continue - next_t += frame_seconds - delay = next_t - time.monotonic() - if delay > 0: - time.sleep(delay) - elif delay < -1.0: - # Fell far behind realtime (e.g. B too large for the GPU); resync - # rather than bursting to catch up. - logger.warning("batched tick loop %.1fs behind realtime; resyncing", -delay) - next_t = time.monotonic() diff --git a/vllm_omni/experimental/fullduplex/personaplex/serving/server.py b/vllm_omni/experimental/fullduplex/personaplex/serving/server.py deleted file mode 100644 index fcb857628..000000000 --- a/vllm_omni/experimental/fullduplex/personaplex/serving/server.py +++ /dev/null @@ -1,539 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""PersonaPlex full-duplex server, compatible with the OFFICIAL Moshi web client. - -Serves the official PersonaPlex web UI (``dist.tgz`` from ``nvidia/personaplex-7b-v1``, -the same bundle ``moshi.server`` downloads) and implements its WebSocket protocol at -``/api/chat`` -- but driven by our native-component engine instead of moshi's server. - -Built on **aiohttp**, mirroring ``moshi.server`` exactly: aiohttp writes+drains each -WebSocket frame promptly, whereas uvicorn's ``websockets`` backend coalesces many small -(~80 ms Opus) frames into a send buffer, which makes browser playback choppy even when -the engine is real-time. aiohttp ships as a moshi dependency, so this adds nothing new. - -Moshi protocol (binary WS messages, first byte = tag): - server -> client ``\\x00`` handshake (ready, after system prompts) - client -> server ``\\x01`` + opus bytes mic audio (Opus @ 24kHz) - server -> client ``\\x01`` + opus bytes agent audio (Opus @ 24kHz) - server -> client ``\\x02`` + utf8 inner-monologue text -Query params on connect: ``text_prompt`` (persona), ``voice_prompt`` (voice file). - -Run (auto-downloads the official UI; open http://:8124/): - HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m \\ - vllm_omni.experimental.fullduplex.personaplex.serving.server --port 8124 - -A raw-PCM endpoint (``/v1/audio/duplex``, JSON ``open`` + float32 frames) remains for -tooling/tests that do not want an Opus dependency. -""" - -from __future__ import annotations - -import argparse -import asyncio -import contextlib -import functools -import json -import logging -import tarfile -import threading -from pathlib import Path - -import numpy as np -import sphn -from aiohttp import WSMsgType, web -from huggingface_hub import hf_hub_download - -from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig -from vllm_omni.experimental.fullduplex.personaplex.engine import PersonaPlexEngine -from vllm_omni.experimental.fullduplex.personaplex.serving.batched import BatchedSessionManager -from vllm_omni.experimental.fullduplex.personaplex.session import ( - PersonaPlexServingSessionState, - PersonaPlexSession, - offer_realtime, -) - -logger = logging.getLogger(__name__) -_HF_REPO = "nvidia/personaplex-7b-v1" - - -def _opus_encode(writer: sphn.OpusStreamWriter, pcm: np.ndarray) -> bytes: - """Encode one PCM frame to Opus bytes, across sphn API versions. - - sphn 0.2.x returns the encoded bytes straight from ``append_pcm``; the older - 0.1.x buffers and requires a ``read_bytes`` drain. Handle both. - """ - out = writer.append_pcm(np.ascontiguousarray(pcm, dtype=np.float32)) - if out is None and hasattr(writer, "read_bytes"): - out = writer.read_bytes() - return out or b"" - - -def _opus_decode(reader: sphn.OpusStreamReader, data: bytes) -> np.ndarray | None: - """Decode Opus bytes to PCM, across sphn API versions (see ``_opus_encode``).""" - out = reader.append_bytes(data) - if out is None and hasattr(reader, "read_pcm"): - out = reader.read_pcm() - if out is None or getattr(out, "shape", (0,))[-1] == 0: - return None - return np.asarray(out, dtype=np.float32) - - -def _official_web_dir() -> Path | None: - """Download + extract the official PersonaPlex web client (dist.tgz).""" - try: - tgz = Path(hf_hub_download(_HF_REPO, "dist.tgz")) - dist = tgz.parent / "dist" - if not dist.exists(): - with tarfile.open(tgz, "r:gz") as tar: - tar.extractall(path=tgz.parent) # noqa: S202 - trusted first-party bundle - return dist if dist.exists() else None - except Exception as exc: # network / auth / layout - logger.warning("official web client unavailable (%s); / will return an error note", exc) - return None - - -class DuplexServer: - """Owns the loaded engine; serves one duplex conversation at a time. - - A capacity-one lease covers the entire WebSocket lifetime on BOTH endpoints: - the engine is a single shared streaming state, so a second concurrent - connection would reset and corrupt the live conversation. Extra connections - are rejected with 1013 (try again later). Engine work (``open``/``feed``, - synchronous CUDA) runs on a worker thread so the event loop stays free for - WebSocket traffic and close handling. - """ - - def __init__(self, config: PersonaPlexConfig) -> None: - self.config = config - self.engine = PersonaPlexEngine(config) - self._active = False # capacity-one session lease (event-loop confined) - self._lease_release_tasks: set[asyncio.Task] = set() - - def load(self) -> None: - self.engine.load() - - async def _lease(self, ws: web.WebSocketResponse) -> bool: - """Try to claim the single-session lease; reject the connection if held.""" - if self._active: - await ws.close(code=1013, message=b"another conversation is active") - return False - self._active = True - return True - - async def _engine_call(self, state: PersonaPlexServingSessionState, fn, *args, **kwargs): - """Run one synchronous engine call on a worker thread, tracked for drain. - - Cancelling the awaiting task does NOT stop the worker thread, and it - marks the asyncio wrapper future cancelled (``done()``) even while the - thread keeps running, so the future itself cannot signal completion. - The worker instead sets a plain ``threading.Event`` when the call truly - finishes; :meth:`_drain_inflight` waits on that event before the lease - is released. Without this, a fast reconnect could acquire the lease and - call ``open()`` while the previous connection's final ``step()`` is - still mutating the shared engine on its thread. - """ - call = functools.partial(fn, *args, **kwargs) - finished = threading.Event() - - def run_and_signal(): - try: - return call() - finally: - finished.set() - - state.inflight = finished - result = await asyncio.get_running_loop().run_in_executor(None, run_and_signal) - state.inflight = None # on cancellation, the event stays referenced for drain - return result - - @staticmethod - async def _drain_inflight(state: PersonaPlexServingSessionState, timeout: float = 5.0) -> bool: - """Wait out any engine call still running on a worker thread.""" - finished = state.inflight - if finished is None or finished.is_set(): - return True - return bool(await asyncio.to_thread(finished.wait, timeout)) - - async def _release_lease_when_finished(self, state: PersonaPlexServingSessionState) -> None: - finished = state.inflight - if finished is not None and not finished.is_set(): - await asyncio.to_thread(finished.wait) - self._active = False - - async def _release_lease_after_drain( - self, - state: PersonaPlexServingSessionState, - *, - timeout: float = 5.0, - ) -> bool: - """Release now when drained, otherwise retain the lease until completion.""" - if await self._drain_inflight(state, timeout): - self._active = False - return True - - logger.error( - "PersonaPlex engine cleanup exceeded %.1fs; retaining the session lease until the worker thread finishes", - timeout, - ) - task = asyncio.create_task(self._release_lease_when_finished(state)) - self._lease_release_tasks.add(task) - task.add_done_callback(self._lease_release_tasks.discard) - return False - - # ---- official Moshi protocol (Opus over aiohttp WS) --------------------- - - async def handle_chat(self, request: web.Request) -> web.WebSocketResponse: - ws = web.WebSocketResponse() - await ws.prepare(request) - if not await self._lease(ws): - return ws - state = PersonaPlexServingSessionState() - try: - persona = (request.query.get("text_prompt") or "").strip() or None - voice = (request.query.get("voice_prompt") or "").strip() or None - sr = self.engine.sample_rate - session = PersonaPlexSession(self.engine, self.config) - # System-prompt prefill is seconds of GPU work; keep it off the loop. - await self._engine_call(state, session.open, voice_prompt=voice, persona=persona) - - reader = sphn.OpusStreamReader(sr) - writer = sphn.OpusStreamWriter(sr) - await ws.send_bytes(b"\x00") # ready handshake - - async def recv_loop() -> None: - try: - async for msg in ws: - if msg.type == WSMsgType.BINARY: - data = msg.data - if data and data[0] == 1: - pcm = _opus_decode(reader, data[1:]) - if pcm is not None: - offer_realtime(state.pcm_queue, pcm) - elif msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.ERROR): - break - finally: - state.closed = True - - async def proc_loop() -> None: - # Drain decoded PCM and step the model, streaming agent Opus + text - # out. The synchronous engine step runs on a worker thread. - while not state.closed: - try: - pcm = await asyncio.wait_for(state.pcm_queue.get(), timeout=0.25) - except asyncio.TimeoutError: - continue - for frame_out in await self._engine_call(state, session.feed, pcm): - if frame_out.audio is not None: - opus = _opus_encode(writer, frame_out.audio) - if opus: - await ws.send_bytes(b"\x01" + opus) - if frame_out.text: - await ws.send_bytes(b"\x02" + frame_out.text.encode("utf8")) - - tasks = [asyncio.create_task(recv_loop()), asyncio.create_task(proc_loop())] - try: - _, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) - for t in pending: - t.cancel() - with contextlib.suppress(asyncio.CancelledError): - await t - finally: - with contextlib.suppress(Exception): - await ws.close() - finally: - # A cancelled feed/open may still be running on its worker thread; - # wait it out BEFORE freeing the lease, or a fast reconnect races - # the shared engine (CUDA illegal access / cross-session bleed). - await self._release_lease_after_drain(state) - return ws - - # ---- simple raw-PCM protocol (no Opus, for tests) ---------------------- - - async def handle_raw(self, request: web.Request) -> web.WebSocketResponse: - ws = web.WebSocketResponse() - await ws.prepare(request) - if not await self._lease(ws): - return ws - session = PersonaPlexSession(self.engine, self.config) - state = PersonaPlexServingSessionState() - opened = False - try: - async for msg in ws: - if msg.type == WSMsgType.TEXT: - req = json.loads(msg.data) - if req.get("type") == "open": - await self._engine_call( - state, session.open, voice_prompt=req.get("voice"), persona=req.get("persona") - ) - opened = True - await ws.send_json({"type": "ready"}) - elif req.get("type") == "close": - if opened: - await self._emit(ws, await self._engine_call(state, session.flush)) - await ws.send_json({"type": "done"}) - break - elif msg.type == WSMsgType.BINARY: - if not opened: - await self._engine_call(state, session.open) - opened = True - pcm = np.frombuffer(msg.data, dtype=np.float32) - await self._emit(ws, await self._engine_call(state, session.feed, pcm)) - elif msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.ERROR): - break - finally: - # Same as handle_chat: wait out any worker-thread engine call - # before freeing the lease (see _engine_call). - await self._release_lease_after_drain(state) - with contextlib.suppress(Exception): - await ws.close() - return ws - - @staticmethod - async def _emit(ws: web.WebSocketResponse, outputs: list) -> None: - for frame_out in outputs: - if frame_out.audio is not None: - await ws.send_bytes(np.ascontiguousarray(frame_out.audio, dtype=np.float32).tobytes()) - if frame_out.text: - await ws.send_json({"type": "text", "text": frame_out.text}) - - -class BatchedDuplexServer: - """Serve ``batch_size`` concurrent duplex conversations on one engine. - - Same wire protocols as :class:`DuplexServer`; each connection claims a slot - from the shared :class:`BatchedSessionManager`, whose stepping thread ticks - all live slots in lockstep at 12.5 Hz wall clock. A connection with a custom - ``text_prompt`` (or reusing a previously used slot) waits while the slot's - system prompt is replayed per-row (voice embeddings + persona), then gets the - ready handshake. Per-connection custom voices are not supported in batched - mode (fixed default voice; ``voice_prompt`` is ignored with a warning). - """ - - def __init__(self, config: PersonaPlexConfig) -> None: - self.config = config - self.engine = PersonaPlexEngine(config) - self.manager: BatchedSessionManager | None = None - self._stop = threading.Event() - self._thread: threading.Thread | None = None - - def load(self) -> None: - self.engine.load() - self.engine.open_batch() - self.manager = BatchedSessionManager(self.engine) - self._thread = threading.Thread( - target=self.manager.run_forever, args=(self._stop,), daemon=True, name="pplex-tick" - ) - self._thread.start() - - def shutdown(self) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=2.0) - - async def _acquire(self, ws: web.WebSocketResponse, persona: str | None): - """Claim a slot and wait until it is live. - - Returns ``PersonaPlexServingSessionState`` (slot + bounded output queue) - or ``None`` when the server is full. Exception-safe: if this coroutine - is cancelled (client gone mid-prefill) or the wait fails, the slot is - released instead of leaking in the PREFILL/LIVE phase. - """ - acquired = self.manager.acquire(persona=persona) - if acquired is None: - await ws.close(code=1013, message=b"all conversation slots busy") - return None - slot, ready_now = acquired - try: - loop = asyncio.get_running_loop() - state = PersonaPlexServingSessionState(slot=slot, epoch=slot.epoch) - ready = asyncio.Event() - with self.manager.lock: - # Bounded handoff: the tick thread produces one frame per 80 ms; - # a client that stops reading gets drop-oldest, not unbounded RAM. - slot.on_output = lambda frame_out: loop.call_soon_threadsafe(offer_realtime, state.out_queue, frame_out) - if ready_now: - ready.set() - else: - slot.on_ready = lambda: loop.call_soon_threadsafe(ready.set) - await ready.wait() - except BaseException: - self.manager.release(slot) - raise - return state - - async def handle_chat(self, request: web.Request) -> web.WebSocketResponse: - ws = web.WebSocketResponse() - await ws.prepare(request) - persona = (request.query.get("text_prompt") or "").strip() or None - if (request.query.get("voice_prompt") or "").strip(): - logger.warning("batched mode ignores per-connection voice_prompt (fixed default voice)") - state = await self._acquire(ws, persona) - if state is None: - return ws - try: - sr = self.engine.sample_rate - reader = sphn.OpusStreamReader(sr) - writer = sphn.OpusStreamWriter(sr) - await ws.send_bytes(b"\x00") # ready handshake - - async def recv_loop() -> None: - try: - async for msg in ws: - if msg.type == WSMsgType.BINARY: - data = msg.data - if data and data[0] == 1: - pcm = _opus_decode(reader, data[1:]) - if pcm is not None: - self.manager.feed(state.slot, pcm, state.epoch) - elif msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.ERROR): - break - finally: - state.closed = True - - async def out_loop() -> None: - while not state.closed: - try: - frame_out = await asyncio.wait_for(state.out_queue.get(), timeout=0.25) - except asyncio.TimeoutError: - continue - if frame_out.audio is not None: - msg = _opus_encode(writer, frame_out.audio) - if msg: - await ws.send_bytes(b"\x01" + msg) - if frame_out.text: - await ws.send_bytes(b"\x02" + frame_out.text.encode("utf8")) - - tasks = [asyncio.create_task(recv_loop()), asyncio.create_task(out_loop())] - _, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) - for t in pending: - t.cancel() - with contextlib.suppress(asyncio.CancelledError): - await t - finally: - self.manager.release(state.slot) - with contextlib.suppress(Exception): - await ws.close() - return ws - - async def handle_raw(self, request: web.Request) -> web.WebSocketResponse: - ws = web.WebSocketResponse() - await ws.prepare(request) - state: PersonaPlexServingSessionState | None = None - sender: asyncio.Task | None = None - - async def pump(outq: asyncio.Queue) -> None: - # Forward slot output continuously (the tick loop produces in realtime). - while True: - frame_out = await outq.get() - if frame_out.audio is not None: - await ws.send_bytes(np.ascontiguousarray(frame_out.audio, dtype=np.float32).tobytes()) - if frame_out.text: - await ws.send_json({"type": "text", "text": frame_out.text}) - - try: - async for msg in ws: - if msg.type == WSMsgType.TEXT: - req = json.loads(msg.data) - if req.get("type") == "open": - if state is not None: - # One slot lease per connection; a second `open` would - # silently leak the first slot and its pump. - await ws.send_json({"type": "error", "error": "session already open"}) - continue - state = await self._acquire(ws, (req.get("persona") or "").strip() or None) - if state is None: - return ws - sender = asyncio.create_task(pump(state.out_queue)) - await ws.send_json({"type": "ready"}) - elif req.get("type") == "close": - await asyncio.sleep(0.3) # let in-flight frames flush via pump - await ws.send_json({"type": "done"}) - break - elif msg.type == WSMsgType.BINARY and state is not None: - self.manager.feed(state.slot, np.frombuffer(msg.data, dtype=np.float32), state.epoch) - elif msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.ERROR): - break - finally: - # Release FIRST and unconditionally: a pump that already failed (e.g. - # peer reset mid-send) re-raises on await and must not leak the slot. - if state is not None: - self.manager.release(state.slot) - if sender is not None: - sender.cancel() - try: - await sender - except asyncio.CancelledError: - pass - except Exception: - logger.debug("raw output pump ended with an error", exc_info=True) - with contextlib.suppress(Exception): - await ws.close() - return ws - - -def create_app(config: PersonaPlexConfig) -> web.Application: - server = BatchedDuplexServer(config) if config.batch_size > 1 else DuplexServer(config) - web_dir = _official_web_dir() - - async def _startup(_app: web.Application) -> None: - logger.info("PersonaPlex duplex server: loading engine (batch_size=%d)", config.batch_size) - server.load() - logger.info("PersonaPlex duplex server: ready (official web: %s)", bool(web_dir)) - - async def _cleanup(_app: web.Application) -> None: - if isinstance(server, BatchedDuplexServer): - server.shutdown() - - app = web.Application() - app.on_startup.append(_startup) - app.on_cleanup.append(_cleanup) - - async def health(_request: web.Request) -> web.Response: - return web.json_response({"status": "ok"}) - - app.router.add_get("/health", health) - app.router.add_get("/api/chat", server.handle_chat) - app.router.add_get("/v1/audio/duplex", server.handle_raw) - - # Serve the official PersonaPlex web client (explicit routes are matched first). - if web_dir is not None: - - async def index(_request: web.Request) -> web.FileResponse: - return web.FileResponse(web_dir / "index.html") - - app.router.add_get("/", index) - app.router.add_static("/", path=str(web_dir), follow_symlinks=True, name="web") - else: - - async def index_missing(_request: web.Request) -> web.Response: - return web.json_response({"error": f"official web client unavailable; check access to {_HF_REPO}"}) - - app.router.add_get("/", index_missing) - - return app - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--host", default="0.0.0.0") - ap.add_argument("--port", type=int, default=8124) - ap.add_argument("--persona", default=None) - ap.add_argument("--voice", default="NATF2.pt") - ap.add_argument( - "--batch-size", - type=int, - default=1, - help="Concurrent conversation slots on one engine (elastic batching when > 1).", - ) - args = ap.parse_args() - logging.basicConfig(level=logging.INFO) - cfg = PersonaPlexConfig( - voice_prompt=args.voice, - batch_size=args.batch_size, - **({"persona": args.persona} if args.persona else {}), - ) - web.run_app(create_app(cfg), host=args.host, port=args.port) - - -if __name__ == "__main__": - main() diff --git a/vllm_omni/experimental/fullduplex/personaplex/session.py b/vllm_omni/experimental/fullduplex/personaplex/session.py deleted file mode 100644 index b3a476f12..000000000 --- a/vllm_omni/experimental/fullduplex/personaplex/session.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""PersonaPlex lockstep session driver — the runnable serving primitive. - -Mirrors JoyVL's ``serving/session.py``: a thin, framework-free driver that owns a -``FrameStepper`` and turns arbitrary-length PCM chunks from a client into exact -80 ms frames, stepping the model one frame at a time. This is the path that -actually runs end to end (offline driver and any WS server build on it); the -``DuplexAdapter`` is a separate demonstration of the generic framework seam. -""" - -from __future__ import annotations - -import asyncio -import contextlib -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any - -import numpy as np - -from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig -from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput, FrameStepper - -if TYPE_CHECKING: - from numpy.typing import NDArray - -# Realtime queue bounds: 80 ms/frame, so 50 frames = 4 s of backlog. Anything -# beyond that means the peer cannot keep up and old frames are already useless. -MAX_PENDING_INPUT = 50 -MAX_PENDING_OUTPUT = 50 - - -def offer_realtime(queue: asyncio.Queue, item: Any) -> None: - """Enqueue with a drop-oldest policy (realtime streams must never grow unbounded).""" - while True: - try: - queue.put_nowait(item) - return - except asyncio.QueueFull: - with contextlib.suppress(asyncio.QueueEmpty): - queue.get_nowait() - - -@dataclass -class PersonaPlexServingSessionState: - """Mutable per-connection serving state (the minicpmo45 ``session.py`` role). - - One instance per WebSocket connection, for both server modes: the - single-session server uses ``pcm_queue`` (decoded mic PCM awaiting the - engine); the batched server uses ``out_queue`` + ``slot``/``epoch`` (engine - output awaiting send, and the claimed slot lease). ``closed`` tears both - pump loops down. - """ - - pcm_queue: asyncio.Queue = field(default_factory=lambda: asyncio.Queue(maxsize=MAX_PENDING_INPUT)) - out_queue: asyncio.Queue = field(default_factory=lambda: asyncio.Queue(maxsize=MAX_PENDING_OUTPUT)) - closed: bool = False - slot: Any = None # batched mode: the claimed serving/batched.py Slot - epoch: int = 0 # batched mode: slot epoch at claim time (stale-guard) - # Single-session mode: completion event of the engine call currently running - # on a worker thread. Task cancellation cannot stop that thread (and marks - # the asyncio future cancelled while the thread runs on), so the server - # waits on this event before releasing its lease (DuplexServer._engine_call). - inflight: Any = None - - -class PersonaPlexSession: - """Drive one PersonaPlex conversation in strict frame lockstep. - - A session buffers incoming PCM and emits exactly one :class:`FrameOutput` per - completed ``frame_size`` window. Partial tails are held until enough samples - arrive (or ``flush()`` pads them out at end of stream). - """ - - def __init__(self, stepper: FrameStepper, config: PersonaPlexConfig | None = None) -> None: - self.stepper = stepper - self.config = config or PersonaPlexConfig() - self.frame_size = stepper.frame_size - self._pending = np.zeros(0, dtype=np.float32) - self._opened = False - - def open(self, voice_prompt: str | None = None, persona: str | None = None) -> None: - """Inject the voice clone + persona and arm the conversation.""" - self.stepper.open_session(voice_prompt=voice_prompt, persona=persona) - self._pending = np.zeros(0, dtype=np.float32) - self._opened = True - - def feed(self, pcm: NDArray[np.float32]) -> list[FrameOutput]: - """Append PCM and step every newly completed frame. - - Returns one :class:`FrameOutput` per frame consumed from the buffer this - call (possibly empty if fewer than ``frame_size`` samples are buffered). - """ - if not self._opened: - raise RuntimeError("PersonaPlexSession.open() must be called before feed()") - samples = np.ascontiguousarray(pcm, dtype=np.float32).reshape(-1) - self._pending = np.concatenate([self._pending, samples]) if self._pending.size else samples - - outputs: list[FrameOutput] = [] - n = self.frame_size - while self._pending.shape[0] >= n: - frame, self._pending = self._pending[:n], self._pending[n:] - outputs.append(self.stepper.step(frame)) - return outputs - - def flush(self) -> list[FrameOutput]: - """Zero-pad and step any buffered tail; call once at end of input.""" - if self._pending.shape[0] == 0: - return [] - pad = self.frame_size - self._pending.shape[0] - frame = np.concatenate([self._pending, np.zeros(pad, dtype=np.float32)]) - self._pending = np.zeros(0, dtype=np.float32) - return [self.stepper.step(frame)] - - def run(self, pcm: NDArray[np.float32]) -> tuple[NDArray[np.float32], str]: - """Convenience for offline use: feed a whole utterance, return (audio, text). - - Concatenates every emitted agent frame and joins the inner-monologue text. - """ - outputs = self.feed(pcm) + self.flush() - chunks = [o.audio for o in outputs if o.audio is not None] - audio = np.concatenate(chunks) if chunks else np.zeros(0, dtype=np.float32) - text = "".join(o.text for o in outputs if o.text) - return audio, text diff --git a/vllm_omni/experimental/fullduplex/model_executor.py b/vllm_omni/model_executor/duplex_sampling.py similarity index 98% rename from vllm_omni/experimental/fullduplex/model_executor.py rename to vllm_omni/model_executor/duplex_sampling.py index 58ba597df..7dbc185d1 100644 --- a/vllm_omni/experimental/fullduplex/model_executor.py +++ b/vllm_omni/model_executor/duplex_sampling.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/__init__.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/__init__.py similarity index 86% rename from vllm_omni/experimental/fullduplex/minicpmo45/__init__.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/__init__.py index 6df010470..b681ad51e 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/__init__.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/__init__.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from .adapter import MiniCPMO45ClientRuntimeConfigError, MiniCPMO45NativeDuplexServingAdapter from .input import MiniCPMO45PcmAppendBuffer diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/adapter.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/adapter.py similarity index 97% rename from vllm_omni/experimental/fullduplex/minicpmo45/adapter.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/adapter.py index 0afffa03c..758beafc9 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/adapter.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/adapter.py @@ -10,12 +10,12 @@ from typing import Any import numpy as np from vllm.multimodal.media import MediaConnector -from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy -from vllm_omni.experimental.fullduplex.openai.protocol import DuplexSessionConfig -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( +from vllm_omni.entrypoints.duplex.protocol import DuplexSessionConfig, native_duplex_opt_in +from vllm_omni.entrypoints.duplex.runtime_adapter import ( ServingRuntimeConfigError, reject_changed_runtime_value, ) +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import MiniCPMO45DuplexPolicy class MiniCPMO45ClientRuntimeConfigError(ServingRuntimeConfigError): @@ -47,7 +47,7 @@ class MiniCPMO45NativeDuplexServingAdapter: @classmethod def is_enabled(cls, config: DuplexSessionConfig) -> bool: - return config.extra_body.get("minicpmo45_native_duplex") is True + return native_duplex_opt_in(config.extra_body) is True @classmethod def validate_client_config(cls, config: DuplexSessionConfig) -> None: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/duplex/capabilities.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/capabilities.py new file mode 100644 index 000000000..668405892 --- /dev/null +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/capabilities.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from __future__ import annotations + +from vllm_omni.entrypoints.duplex.protocol import DuplexCapabilities + + +def minicpmo45_native_capabilities(*, max_sessions: int = 1) -> DuplexCapabilities: + """The MiniCPM-o 4.5 native-duplex preset of the generic capability payload.""" + supports_multi_session = max_sessions > 1 + return DuplexCapabilities( + supports_model_native_turn_policy=True, + supports_barge_in=True, + supports_input_append=True, + supports_replace_latest_chunk=False, + supports_reencode_context=False, + supports_turn_commit_only=False, + supports_kv_lease=False, + supports_core_kv_lease=False, + supports_model_internal_state=True, + supports_stage_resumption=True, + supports_scheduler_native_append=False, + supports_core_resumable_request=True, + supports_stage_connector_handoff=True, + supports_independent_io_streams=True, + supports_realtime_endpoint=True, + supports_multi_session=supports_multi_session, + supports_multi_session_same_replica=supports_multi_session, + supports_session_lease=True, + supports_session_resume=True, + session_admission_mode="engine_managed", + supports_audio_truncate=True, + requires_model_runner_kv=True, + requires_native_stage_role=True, + implementation_level="model_native_duplex", + adapter_patterns=["scheduler_data_plane"], + input_modes=["append_audio_chunk"], + signal_sources=["model_native", "client_event", "server_policy"], + stage_handoff_transport="scheduler_data_plane", + chunk_period_ms=1000, + # Barge-in latency depends on client chunking and lacks hardware E2E + # measurement, so do not advertise an invented target. + target_barge_in_latency_ms=None, + ) + + +__all__ = ["minicpmo45_native_capabilities"] diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/compat.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/compat.py similarity index 92% rename from vllm_omni/experimental/fullduplex/minicpmo45/compat.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/compat.py index 9b44bed4c..9a4bcf9b0 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/compat.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/compat.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations from typing import Any diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/data_plane.py similarity index 99% rename from vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/data_plane.py index d18c0f4a9..d4063ff3f 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/data_plane.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations from collections.abc import Callable, Iterator, Mapping @@ -6,10 +9,10 @@ from dataclasses import dataclass, field import numpy as np from vllm.logger import init_logger -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex.contracts import ( duplex_resource_request_belongs_to_session, ) -from vllm_omni.experimental.fullduplex.output import get_duplex_output_decision +from vllm_omni.outputs.duplex import get_duplex_output_decision logger = init_logger(__name__) diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/input.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/input.py similarity index 99% rename from vllm_omni/experimental/fullduplex/minicpmo45/input.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/input.py index 248e857e8..f7d86f1d7 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/input.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/input.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import base64 diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/policy.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/policy.py similarity index 100% rename from vllm_omni/experimental/fullduplex/minicpmo45/policy.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/policy.py diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/runtime.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/runtime.py similarity index 98% rename from vllm_omni/experimental/fullduplex/minicpmo45/runtime.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/runtime.py index ecd634f73..c8e746ce7 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/runtime.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/runtime.py @@ -9,13 +9,13 @@ from typing import Any, cast from vllm.sampling_params import SamplingParams -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import ( DuplexAppendPlan, DuplexInputMode, DuplexOutputAction, DuplexOutputDecision, ) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence _DUPLEX_CHUNK_SAMPLES = 16000 _DUPLEX_SAMPLES_PER_AUDIO_TOKEN = 1600 diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/serving_adapter.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/serving_adapter.py similarity index 83% rename from vllm_omni/experimental/fullduplex/minicpmo45/serving_adapter.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/serving_adapter.py index e811d73e5..ea51662a9 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/serving_adapter.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/serving_adapter.py @@ -1,19 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations from collections.abc import Callable, Mapping from typing import Any -from vllm_omni.experimental.fullduplex.minicpmo45.adapter import ( +from vllm_omni.entrypoints.duplex.protocol import DuplexCapabilities +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.adapter import ( MiniCPMO45NativeDuplexServingAdapter, ) -from vllm_omni.experimental.fullduplex.minicpmo45.data_plane import ( +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import ( + minicpmo45_native_capabilities, +) +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.data_plane import ( MiniCPMO45DataPlaneContext, MiniCPMO45DataPlaneSession, ) -from vllm_omni.experimental.fullduplex.minicpmo45.session import ( +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.session import ( MiniCPMO45ServingSessionState, ) -from vllm_omni.experimental.fullduplex.openai.protocol import DuplexCapabilities EncodeAudio = Callable[[object, int, str, float | None], str | None] @@ -49,7 +55,7 @@ class MiniCPMO45ServingRuntimeAdapter: @staticmethod def capabilities(*, max_sessions: int) -> DuplexCapabilities: - return DuplexCapabilities.minicpmo45_native(max_sessions=max_sessions) + return minicpmo45_native_capabilities(max_sessions=max_sessions) @staticmethod def validate_client_extra_body(extra_body: object) -> None: diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/session.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/session.py similarity index 91% rename from vllm_omni/experimental/fullduplex/minicpmo45/session.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/session.py index 1848ee81b..40c42cb81 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/session.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/session.py @@ -1,10 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import asyncio from collections.abc import Awaitable, Callable from dataclasses import dataclass, field -from vllm_omni.experimental.fullduplex.minicpmo45.input import ( +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.input import ( MiniCPMO45PcmAppendBuffer, ) diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py similarity index 99% rename from vllm_omni/experimental/fullduplex/minicpmo45/stage0.py rename to vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py index fbea82188..6d9800025 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py @@ -13,7 +13,7 @@ from typing import Any import numpy as np -from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import MiniCPMO45DuplexPolicy _MINICPMO45_SPECIAL_TOKEN_FIELDS = MiniCPMO45DuplexPolicy.SPECIAL_TOKEN_FIELDS _MINICPMO45_OPTIONAL_TOKEN_FIELDS = MiniCPMO45DuplexPolicy.OPTIONAL_TOKEN_FIELDS @@ -535,7 +535,7 @@ class MiniCPMO45Stage0DuplexRuntime: @staticmethod def _decode_ref_audio_from_session_config(session_config: dict[str, Any]) -> Any | None: - from vllm_omni.experimental.fullduplex.minicpmo45.input import decode_native_ref_audio_from_config + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.input import decode_native_ref_audio_from_config return decode_native_ref_audio_from_config({"extra_body": session_config}) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py index f2733f3b7..e9ed05e75 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project # Adapted from: # https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/main/modeling_minicpmo.py # @@ -34,8 +34,8 @@ from vllm.sequence import IntermediateTensors from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata -from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45DuplexPolicy -from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow +from vllm_omni.model_executor.duplex_sampling import DuplexSamplingRow +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import MiniCPMO45DuplexPolicy from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( MiniCPMO45OmniLLMDummyInputsBuilder, MiniCPMO45OmniLLMMultiModalProcessor, @@ -85,7 +85,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp # Store configs self.config = config self.multimodal_config = multimodal_config - from vllm_omni.experimental.fullduplex.minicpmo45.compat import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.compat import ( patch_minicpmo_remote_config, ) @@ -338,7 +338,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp session_key = (session_id, incarnation) state = helper.sessions.get(session_key) if state is None: - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import ( _MiniCPMO45Stage0SessionState, ) @@ -456,7 +456,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp helper = getattr(self, "_minicpmo45_duplex_data_plane_helper", None) if helper is not None: return helper - from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import MiniCPMO45Stage0DuplexRuntime + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import MiniCPMO45Stage0DuplexRuntime model_path = getattr(getattr(self.vllm_config, "model_config", None), "model", None) device = str(self._module_device(self.thinker if self.thinker is not None else self)) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index af9401872..b0fa9d203 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project # Adapted from: # https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/main/modeling_minicpmo.py # @@ -3941,7 +3941,7 @@ class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, S else: text_config = Qwen2Config.from_dict(config_dict) llm_arch = "Qwen2ForCausalLM" - from vllm_omni.experimental.fullduplex.minicpmo45.compat import ( + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.compat import ( patch_minicpmo_remote_config, ) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index 369219317..7c87f1a8b 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -27,7 +27,7 @@ from vllm.model_executor.models.llama import LlamaModel from vllm.model_executor.models.utils import maybe_prefix from vllm.v1.sample.sampler import Sampler -from vllm_omni.experimental.fullduplex.engine.intermediate import get_tts_handoff +from vllm_omni.engine.duplex.intermediate import get_tts_handoff from vllm_omni.model_executor.models.minicpmo_4_5 import MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.platforms import current_omni_platform diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py index 51898f27a..0ca61a346 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py @@ -24,9 +24,11 @@ MINICPMO_4_5_PIPELINE = PipelineConfig( model_type="minicpmo_4_5", default_deploy_config_name="minicpmo_4_5.yaml", model_arch="MiniCPMO45OmniForConditionalGeneration", - duplex_runtime_extension=("vllm_omni.experimental.fullduplex.minicpmo45.runtime.MiniCPMO45DuplexRuntimeExtension"), + duplex_runtime_extension=( + "vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime.MiniCPMO45DuplexRuntimeExtension" + ), duplex_serving_adapter=( - "vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter" + "vllm_omni.model_executor.models.minicpmo_4_5.duplex.serving_adapter.MiniCPMO45ServingRuntimeAdapter" ), duplex_control_enabled=True, # MiniCPM-o 4.5's HF config.json reports `model_type="minicpmo"` and diff --git a/vllm_omni/experimental/fullduplex/nemotron_voicechat/__init__.py b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/__init__.py similarity index 60% rename from vllm_omni/experimental/fullduplex/nemotron_voicechat/__init__.py rename to vllm_omni/model_executor/models/nemotron_voicechat/duplex/__init__.py index ce738cce6..ab79ab4d2 100644 --- a/vllm_omni/experimental/fullduplex/nemotron_voicechat/__init__.py +++ b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/__init__.py @@ -1,4 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Native Full-Duplex integration for NVIDIA NemotronLabs VoiceChat.""" diff --git a/vllm_omni/experimental/fullduplex/nemotron_voicechat/data_plane.py b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/data_plane.py similarity index 98% rename from vllm_omni/experimental/fullduplex/nemotron_voicechat/data_plane.py rename to vllm_omni/model_executor/models/nemotron_voicechat/duplex/data_plane.py index 7cc2a43ef..0fb3f93c7 100644 --- a/vllm_omni/experimental/fullduplex/nemotron_voicechat/data_plane.py +++ b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/data_plane.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Output projection for Nemotron VoiceChat's independent channels.""" @@ -15,10 +15,10 @@ import numpy as np import torch from vllm.logger import init_logger -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex.contracts import ( duplex_resource_request_belongs_to_session, ) -from vllm_omni.experimental.fullduplex.output import ( +from vllm_omni.outputs.duplex import ( get_duplex_output_decision, ) diff --git a/vllm_omni/experimental/fullduplex/nemotron_voicechat/input.py b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/input.py similarity index 99% rename from vllm_omni/experimental/fullduplex/nemotron_voicechat/input.py rename to vllm_omni/model_executor/models/nemotron_voicechat/duplex/input.py index e69ebf4f4..860671cb2 100644 --- a/vllm_omni/experimental/fullduplex/nemotron_voicechat/input.py +++ b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/input.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """80 ms PCM packetization for Nemotron VoiceChat native duplex.""" diff --git a/vllm_omni/experimental/fullduplex/nemotron_voicechat/runtime.py b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/runtime.py similarity index 95% rename from vllm_omni/experimental/fullduplex/nemotron_voicechat/runtime.py rename to vllm_omni/model_executor/models/nemotron_voicechat/duplex/runtime.py index 99de30aa7..b017246bd 100644 --- a/vllm_omni/experimental/fullduplex/nemotron_voicechat/runtime.py +++ b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/runtime.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Engine policy for Nemotron VoiceChat's frame-locked duplex timeline.""" @@ -9,14 +9,14 @@ from typing import Any from vllm.sampling_params import RequestOutputKind, SamplingParams -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex.contracts import ( DuplexAppendPlan, DuplexInputMode, DuplexOutputAction, DuplexOutputDecision, ) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence -from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import ( +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.model_executor.models.nemotron_voicechat.duplex.input import ( decode_pcm_f32le, ) diff --git a/vllm_omni/experimental/fullduplex/nemotron_voicechat/serving_adapter.py b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/serving_adapter.py similarity index 97% rename from vllm_omni/experimental/fullduplex/nemotron_voicechat/serving_adapter.py rename to vllm_omni/model_executor/models/nemotron_voicechat/duplex/serving_adapter.py index 2505b3766..16a33ec01 100644 --- a/vllm_omni/experimental/fullduplex/nemotron_voicechat/serving_adapter.py +++ b/vllm_omni/model_executor/models/nemotron_voicechat/duplex/serving_adapter.py @@ -12,18 +12,18 @@ from collections.abc import Callable, Mapping from copy import deepcopy from typing import Any -from vllm_omni.experimental.fullduplex.minicpmo45.session import ( +from vllm_omni.entrypoints.duplex.protocol import DuplexCapabilities +from vllm_omni.entrypoints.duplex.runtime_adapter import ServingRuntimeConfigError +from vllm_omni.model_executor.models.minicpmo_4_5.duplex.session import ( MiniCPMO45ServingSessionState, ) -from vllm_omni.experimental.fullduplex.nemotron_voicechat.data_plane import ( +from vllm_omni.model_executor.models.nemotron_voicechat.duplex.data_plane import ( NemotronVoiceChatDataPlaneContext, NemotronVoiceChatDataPlaneSession, ) -from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import ( +from vllm_omni.model_executor.models.nemotron_voicechat.duplex.input import ( NemotronVoiceChatPcmAppendBuffer, ) -from vllm_omni.experimental.fullduplex.openai.protocol import DuplexCapabilities -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ServingRuntimeConfigError EncodeAudio = Callable[[object, int, str, float | None], str | None] diff --git a/vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_thinker.py b/vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_thinker.py index 317107317..0701a8b7b 100644 --- a/vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_thinker.py +++ b/vllm_omni/model_executor/models/nemotron_voicechat/nemotron_voicechat_thinker.py @@ -52,7 +52,7 @@ from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_p from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata -from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow +from vllm_omni.model_executor.duplex_sampling import DuplexSamplingRow from vllm_omni.model_executor.models.nemotron_voicechat.runtime_info import ( merge_runtime_info, require_request_id, @@ -497,7 +497,7 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState, f"Nemotron VoiceChat duplex input sequence moved backwards: {source_input_seq} < {last_input_seq}" ) - from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import ( + from vllm_omni.model_executor.models.nemotron_voicechat.duplex.input import ( decode_pcm_f32le, ) diff --git a/vllm_omni/model_executor/models/nemotron_voicechat/pipeline.py b/vllm_omni/model_executor/models/nemotron_voicechat/pipeline.py index 1bb1e19b8..3cbe12742 100644 --- a/vllm_omni/model_executor/models/nemotron_voicechat/pipeline.py +++ b/vllm_omni/model_executor/models/nemotron_voicechat/pipeline.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """NemotronVoiceChat pipeline: thinker (speech -> frame-locked text) -> talker (text timeline -> 31-quantizer RVQ code stacks) -> code2wav (codes -> 22.05 kHz PCM). @@ -32,10 +32,10 @@ NEMOTRON_VOICECHAT_PIPELINE = PipelineConfig( model_type="nemotron_voicechat", model_arch="NemotronVoiceChatThinkerForConditionalGeneration", duplex_runtime_extension=( - "vllm_omni.experimental.fullduplex.nemotron_voicechat.runtime.NemotronVoiceChatDuplexRuntimeExtension" + "vllm_omni.model_executor.models.nemotron_voicechat.duplex.runtime.NemotronVoiceChatDuplexRuntimeExtension" ), duplex_serving_adapter=( - "vllm_omni.experimental.fullduplex.nemotron_voicechat.serving_adapter.NemotronVoiceChatServingRuntimeAdapter" + "vllm_omni.model_executor.models.nemotron_voicechat.duplex.serving_adapter.NemotronVoiceChatServingRuntimeAdapter" ), duplex_control_enabled=True, # Named after the alias key so the deploy-yaml stem substring-matches the diff --git a/vllm_omni/model_executor/models/personaplex/duplex/__init__.py b/vllm_omni/model_executor/models/personaplex/duplex/__init__.py new file mode 100644 index 000000000..d71a41c5d --- /dev/null +++ b/vllm_omni/model_executor/models/personaplex/duplex/__init__.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""PersonaPlex full-duplex integration. + +PersonaPlex (``nvidia/personaplex-7b-v1``) is a Moshi finetune: a pure-lockstep +speech-to-speech model. This package plugs it into the generic duplex serving +stack through the standard plugin seams (``duplex_serving_adapter`` / +``duplex_runtime_extension`` dotted strings in the model's ``pipeline.py``): + +- :class:`PersonaPlexConfig` immutable session config (voice / persona / sampling) +- :class:`PersonaPlexServingRuntimeAdapter` the ``ServingRuntimeAdapter`` impl +- :class:`PersonaPlexDuplexRuntimeExtension` the engine ``DuplexRuntimeExtension`` +- :class:`PersonaPlexStage0DuplexRuntime` Stage 0 session state and prefill +- :class:`PersonaPlexPcmAppendBuffer` PCM input framing +""" + +from .config import ( + PersonaPlexConfig, +) +from .input import ( + PersonaPlexPcmAppendBuffer, +) +from .policy import PrefillStep +from .runtime_extension import ( + PersonaPlexDuplexRuntimeExtension, +) +from .serving_adapter import ( + PersonaPlexServingRuntimeAdapter, +) +from .stage0 import ( + PersonaPlexStage0DuplexRuntime, +) + +__all__ = [ + "PersonaPlexConfig", + "PersonaPlexDuplexRuntimeExtension", + "PersonaPlexPcmAppendBuffer", + "PersonaPlexServingRuntimeAdapter", + "PersonaPlexStage0DuplexRuntime", + "PrefillStep", +] diff --git a/vllm_omni/experimental/fullduplex/personaplex/config.py b/vllm_omni/model_executor/models/personaplex/duplex/config.py similarity index 97% rename from vllm_omni/experimental/fullduplex/personaplex/config.py rename to vllm_omni/model_executor/models/personaplex/duplex/config.py index 1cfa4b736..13f807b3e 100644 --- a/vllm_omni/experimental/fullduplex/personaplex/config.py +++ b/vllm_omni/model_executor/models/personaplex/duplex/config.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Configuration for the PersonaPlex full-duplex backend. PersonaPlex (``nvidia/personaplex-7b-v1``) is a Moshi finetune: a pure-lockstep diff --git a/vllm_omni/experimental/fullduplex/personaplex/data_plane.py b/vllm_omni/model_executor/models/personaplex/duplex/data_plane.py similarity index 98% rename from vllm_omni/experimental/fullduplex/personaplex/data_plane.py rename to vllm_omni/model_executor/models/personaplex/duplex/data_plane.py index 96e1f9e20..c6c293c17 100644 --- a/vllm_omni/experimental/fullduplex/personaplex/data_plane.py +++ b/vllm_omni/model_executor/models/personaplex/duplex/data_plane.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -8,7 +8,7 @@ from dataclasses import dataclass import numpy as np -from vllm_omni.experimental.fullduplex.engine.contracts import ( +from vllm_omni.engine.duplex.contracts import ( duplex_resource_request_belongs_to_session, ) diff --git a/vllm_omni/experimental/fullduplex/personaplex/input.py b/vllm_omni/model_executor/models/personaplex/duplex/input.py similarity index 98% rename from vllm_omni/experimental/fullduplex/personaplex/input.py rename to vllm_omni/model_executor/models/personaplex/duplex/input.py index 2fe57442a..e78d6b707 100644 --- a/vllm_omni/experimental/fullduplex/personaplex/input.py +++ b/vllm_omni/model_executor/models/personaplex/duplex/input.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations diff --git a/vllm_omni/experimental/fullduplex/personaplex/policy.py b/vllm_omni/model_executor/models/personaplex/duplex/policy.py similarity index 96% rename from vllm_omni/experimental/fullduplex/personaplex/policy.py rename to vllm_omni/model_executor/models/personaplex/duplex/policy.py index 8e6062d0d..a22b81fe8 100644 --- a/vllm_omni/experimental/fullduplex/personaplex/policy.py +++ b/vllm_omni/model_executor/models/personaplex/duplex/policy.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """PersonaPlex frame/token contract (the model *policy*, no engine state). This mirrors the ``minicpmo45`` model-package layout from PR #3907: the token diff --git a/vllm_omni/experimental/fullduplex/personaplex/runtime_extension.py b/vllm_omni/model_executor/models/personaplex/duplex/runtime_extension.py similarity index 95% rename from vllm_omni/experimental/fullduplex/personaplex/runtime_extension.py rename to vllm_omni/model_executor/models/personaplex/duplex/runtime_extension.py index 036103a43..a30ac41b9 100644 --- a/vllm_omni/experimental/fullduplex/personaplex/runtime_extension.py +++ b/vllm_omni/model_executor/models/personaplex/duplex/runtime_extension.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -9,12 +9,12 @@ from typing import Any from vllm.sampling_params import SamplingParams -from vllm_omni.experimental.fullduplex.engine.duplex_runtime import ( +from vllm_omni.engine.duplex.messages import DuplexFence +from vllm_omni.engine.duplex.runtime import ( DuplexAppendPlan, DuplexInputMode, DuplexOutputDecision, ) -from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence _FRAME_BYTES = 1920 * 4 diff --git a/vllm_omni/experimental/fullduplex/personaplex/serving_adapter.py b/vllm_omni/model_executor/models/personaplex/duplex/serving_adapter.py similarity index 93% rename from vllm_omni/experimental/fullduplex/personaplex/serving_adapter.py rename to vllm_omni/model_executor/models/personaplex/duplex/serving_adapter.py index d4f920d44..d27360fd0 100644 --- a/vllm_omni/experimental/fullduplex/personaplex/serving_adapter.py +++ b/vllm_omni/model_executor/models/personaplex/duplex/serving_adapter.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -10,19 +10,19 @@ from dataclasses import dataclass, field from pathlib import PurePath from typing import Any -from vllm_omni.experimental.fullduplex.openai.protocol import ( +from vllm_omni.entrypoints.duplex.protocol import ( DuplexCapabilities, ) -from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ( +from vllm_omni.entrypoints.duplex.runtime_adapter import ( ServingRuntimeConfigError, reject_changed_runtime_value, ) -from vllm_omni.experimental.fullduplex.personaplex.config import DEFAULT_PERSONA -from vllm_omni.experimental.fullduplex.personaplex.data_plane import ( +from vllm_omni.model_executor.models.personaplex.duplex.config import DEFAULT_PERSONA +from vllm_omni.model_executor.models.personaplex.duplex.data_plane import ( PersonaPlexDataPlaneContext, PersonaPlexDataPlaneSession, ) -from vllm_omni.experimental.fullduplex.personaplex.input import ( +from vllm_omni.model_executor.models.personaplex.duplex.input import ( PersonaPlexPcmAppendBuffer, ) @@ -32,6 +32,9 @@ _PRIVATE_RUNTIME_CONFIG_KEYS = frozenset( { "personaplex_prefill_slots", "personaplex_model_path", + # PersonaPlex is always model-native; the per-session opt-in knob + # (canonical name and its deprecated alias) is server-owned here. + "native_duplex", "minicpmo45_native_duplex", } ) @@ -166,7 +169,7 @@ class PersonaPlexServingRuntimeAdapter: raise ServingRuntimeConfigError("PersonaPlex model path is unavailable") voice = cls._voice_name(getattr(config, "voice", None)) instructions = getattr(config, "instructions", None) or DEFAULT_PERSONA - from vllm_omni.experimental.fullduplex.personaplex.stage0 import ( + from vllm_omni.model_executor.models.personaplex.duplex.stage0 import ( personaplex_prefill_slots, ) diff --git a/vllm_omni/experimental/fullduplex/personaplex/stage0.py b/vllm_omni/model_executor/models/personaplex/duplex/stage0.py similarity index 99% rename from vllm_omni/experimental/fullduplex/personaplex/stage0.py rename to vllm_omni/model_executor/models/personaplex/duplex/stage0.py index a11c5dc40..58d476bc5 100644 --- a/vllm_omni/experimental/fullduplex/personaplex/stage0.py +++ b/vllm_omni/model_executor/models/personaplex/duplex/stage0.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -14,7 +14,7 @@ from typing import Any import numpy as np -from vllm_omni.experimental.fullduplex.personaplex.policy import ( +from vllm_omni.model_executor.models.personaplex.duplex.policy import ( AUDIO_SILENCE_FRAME_CNT, SILENCE_TOKENS, SINE_TOKENS, diff --git a/vllm_omni/model_executor/models/personaplex/personaplex_code2wav.py b/vllm_omni/model_executor/models/personaplex/personaplex_code2wav.py index a5712aa40..63456b4f9 100644 --- a/vllm_omni/model_executor/models/personaplex/personaplex_code2wav.py +++ b/vllm_omni/model_executor/models/personaplex/personaplex_code2wav.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Stage-1 Code2Wav model for PersonaPlex (Moshi finetune). This is the ``LLM_GENERATION`` codec stage of the 2-stage PersonaPlex pipeline. @@ -23,7 +23,7 @@ Layout contract (mirrors ``Qwen3TTSCode2Wav``): The Mimi decoder is the transformers ``MimiModel`` (kyutai/mimi weights), not in vLLM's safetensors loader, so they are loaded eagerly in ``load_weights`` part of the vLLM weights iterator (the codec owns its own checkpoint; see the -experimental fullduplex package for the same pattern). +``duplex`` subpackage of this model folder for the same pattern). """ from __future__ import annotations diff --git a/vllm_omni/model_executor/models/personaplex/personaplex_talker.py b/vllm_omni/model_executor/models/personaplex/personaplex_talker.py index 5e56e7a7c..0ea3e4f12 100644 --- a/vllm_omni/model_executor/models/personaplex/personaplex_talker.py +++ b/vllm_omni/model_executor/models/personaplex/personaplex_talker.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """PersonaPlex talker: the temporal transformer as a vLLM-native omni AR stage. This is the stage-0 (``LLM_AR``) model the OmniGPUModelRunner drives. It composes @@ -420,7 +420,7 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module): runtime = getattr(self, "_personaplex_duplex_stage0_runtime", None) if runtime is not None: return runtime - from vllm_omni.experimental.fullduplex.personaplex.stage0 import ( + from vllm_omni.model_executor.models.personaplex.duplex.stage0 import ( PersonaPlexStage0DuplexRuntime, ) diff --git a/vllm_omni/model_executor/models/personaplex/pipeline.py b/vllm_omni/model_executor/models/personaplex/pipeline.py index 1865d4b3a..ac34bbd23 100644 --- a/vllm_omni/model_executor/models/personaplex/pipeline.py +++ b/vllm_omni/model_executor/models/personaplex/pipeline.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """PersonaPlex pipeline: Talker (AR decode -> Mimi codebooks) -> Code2Wav (codebooks -> 24 kHz PCM). PersonaPlex is a Moshi finetune (full-duplex speech-to-speech). For offline/batch @@ -32,10 +32,10 @@ PERSONAPLEX_PIPELINE = PipelineConfig( model_arch="PersonaPlexTalkerForConditionalGeneration", default_deploy_config_name="personaplex.yaml", duplex_runtime_extension=( - "vllm_omni.experimental.fullduplex.personaplex.runtime_extension.PersonaPlexDuplexRuntimeExtension" + "vllm_omni.model_executor.models.personaplex.duplex.runtime_extension.PersonaPlexDuplexRuntimeExtension" ), duplex_serving_adapter=( - "vllm_omni.experimental.fullduplex.personaplex.serving_adapter.PersonaPlexServingRuntimeAdapter" + "vllm_omni.model_executor.models.personaplex.duplex.serving_adapter.PersonaPlexServingRuntimeAdapter" ), duplex_control_enabled=True, stages=( diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py index b293a95c8..c1cbd14bc 100644 --- a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py @@ -10,7 +10,7 @@ import torch from vllm.inputs import TextPrompt from vllm_omni.data_entry_keys import CodesStruct, MetaStruct, OmniPayloadStruct -from vllm_omni.experimental.fullduplex.engine.intermediate import ( +from vllm_omni.engine.duplex.intermediate import ( build_duplex_intermediate_buffer, set_ref_audio, set_tts_handoff, @@ -95,7 +95,7 @@ def _extract_native_runtime_ref_audio(data_plane_metadata): runtime_config = data_plane_metadata.get("runtime_config") if not isinstance(runtime_config, dict): return None - from vllm_omni.experimental.fullduplex.minicpmo45.input import decode_native_ref_audio_from_config + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.input import decode_native_ref_audio_from_config waveform = decode_native_ref_audio_from_config({"extra_body": runtime_config}) if waveform is None: diff --git a/vllm_omni/experimental/fullduplex/output.py b/vllm_omni/outputs/duplex.py similarity index 84% rename from vllm_omni/experimental/fullduplex/output.py rename to vllm_omni/outputs/duplex.py index bfe2243f9..7f51621f6 100644 --- a/vllm_omni/experimental/fullduplex/output.py +++ b/vllm_omni/outputs/duplex.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -7,7 +7,7 @@ from typing import Any from vllm_omni.outputs import OmniRequestOutput -DUPLEX_OUTPUT_DECISION_KEY = "_vllm_omni.experimental.fullduplex.duplex_output_decision" +DUPLEX_OUTPUT_DECISION_KEY = "_vllm_omni.duplex.duplex_output_decision" def attach_duplex_output_decision( diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index 36a3499cc..734178e01 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -46,7 +46,7 @@ from vllm_ascend.worker.model_runner_v1 import graph_capture from vllm_omni.data_entry_keys import flatten_payload from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager from vllm_omni.distributed.omni_connectors.utils.config import stage_sends_async_output -from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRunnerMixin +from vllm_omni.model_executor.duplex_sampling import DuplexSamplingRunnerMixin from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner from vllm_omni.utils.mm_outputs import build_mm_cpu, partition_payload_list, to_payload_element diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index cd03cf3fc..d293f6312 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -45,7 +45,7 @@ from vllm.v1.worker.utils import is_residual_scattered_for_sp from vllm_omni.data_entry_keys import flatten_payload from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager from vllm_omni.distributed.omni_connectors.utils.config import stage_sends_async_output -from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRunnerMixin +from vllm_omni.model_executor.duplex_sampling import DuplexSamplingRunnerMixin from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.utils.mm_outputs import ( build_mm_cpu,