[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: psv666 <2693925048@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
59 files changed
+1908
-155
No files matched your search
@@ -8,7 +8,7 @@
|
|||||||
# NPU naming: {queue}_{resource_class}
|
# NPU naming: {queue}_{resource_class}
|
||||||
# a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
|
# a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
|
||||||
# a2b3_npu_4 — ascend-a2b3, resource_class npu-4
|
# a2b3_npu_4 — ascend-a2b3, resource_class npu-4
|
||||||
# a3_npu_2 — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
|
# a3_npu_2 — ascend-a3, resource_class npu-2
|
||||||
|
|
||||||
x-npu-k8s-plugins: &npu_k8s_plugins
|
x-npu-k8s-plugins: &npu_k8s_plugins
|
||||||
- kubernetes:
|
- kubernetes:
|
||||||
|
|||||||
@@ -139,7 +139,7 @@ steps:
|
|||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 50
|
||||||
commands:
|
commands:
|
||||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||||
mirror_hardwares: h100_2
|
mirror_hardwares: h100_1
|
||||||
|
|
||||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
@@ -148,10 +148,10 @@ steps:
|
|||||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
- tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
- tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||||
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
||||||
- tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
|
- tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
|
||||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
||||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
||||||
- tests/helpers/minicpmo_4_5_duplex.py
|
- tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
|
||||||
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
||||||
- vllm_omni/experimental/fullduplex/
|
- vllm_omni/experimental/fullduplex/
|
||||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||||
@@ -164,7 +164,7 @@ steps:
|
|||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 50
|
||||||
commands:
|
commands:
|
||||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||||
mirror_hardwares: h100_2
|
mirror_hardwares: h100_1
|
||||||
|
|
||||||
- label: "TTS · VoxCPM2 Test"
|
- label: "TTS · VoxCPM2 Test"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ steps:
|
|||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 50
|
||||||
commands:
|
commands:
|
||||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
|
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
|
||||||
mirror_hardwares: h100_2
|
mirror_hardwares: h100_1
|
||||||
|
|
||||||
- label: ":full_moon: Omni · Doc Test with L4"
|
- label: ":full_moon: Omni · Doc Test with L4"
|
||||||
timeout_in_minutes: 90
|
timeout_in_minutes: 90
|
||||||
@@ -50,6 +50,19 @@ steps:
|
|||||||
exit $$EXIT
|
exit $$EXIT
|
||||||
mirror_hardwares: h100_2
|
mirror_hardwares: h100_2
|
||||||
|
|
||||||
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
commands:
|
||||||
|
- export SEED_TTS_WER_EVAL=1
|
||||||
|
- export SEED_TTS_EVAL_DEVICE=cuda:1
|
||||||
|
- |
|
||||||
|
set +e
|
||||||
|
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model
|
||||||
|
EXIT=$$?
|
||||||
|
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||||
|
exit $$EXIT
|
||||||
|
mirror_hardwares: h100_2
|
||||||
|
|
||||||
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
||||||
key: nightly-omni-performance-no-async-chunk
|
key: nightly-omni-performance-no-async-chunk
|
||||||
timeout_in_minutes: 300
|
timeout_in_minutes: 300
|
||||||
@@ -102,6 +115,32 @@ steps:
|
|||||||
exit $$EXIT
|
exit $$EXIT
|
||||||
mirror_hardwares: h100_3
|
mirror_hardwares: h100_3
|
||||||
|
|
||||||
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
||||||
|
key: nightly-omni-performance-minicpmo-4-5
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
commands:
|
||||||
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||||
|
- |
|
||||||
|
set +e
|
||||||
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||||
|
EXIT=$$?
|
||||||
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||||
|
exit $$EXIT
|
||||||
|
mirror_hardwares: h100_1
|
||||||
|
|
||||||
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
||||||
|
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
commands:
|
||||||
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||||
|
- |
|
||||||
|
set +e
|
||||||
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
||||||
|
EXIT=$$?
|
||||||
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||||
|
exit $$EXIT
|
||||||
|
mirror_hardwares: h100_1
|
||||||
|
|
||||||
- label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
|
- label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
|
||||||
timeout_in_minutes: 45
|
timeout_in_minutes: 45
|
||||||
commands:
|
commands:
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ steps:
|
|||||||
timeout 20m bash -c "
|
timeout 20m bash -c "
|
||||||
pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||||
"
|
"
|
||||||
mirror_hardwares: h100_2
|
mirror_hardwares: h100_1
|
||||||
|
|
||||||
- label: "Omni · MiniCPM-o 4.5 Online Test"
|
- label: "Omni · MiniCPM-o 4.5 Online Test"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
@@ -183,7 +183,7 @@ steps:
|
|||||||
timeout 20m bash -c "
|
timeout 20m bash -c "
|
||||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||||
"
|
"
|
||||||
mirror_hardwares: h100_2
|
mirror_hardwares: h100_1
|
||||||
|
|
||||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
@@ -192,10 +192,10 @@ steps:
|
|||||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
- tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
- tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||||
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
||||||
- tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
|
- tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
|
||||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
||||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
||||||
- tests/helpers/minicpmo_4_5_duplex.py
|
- tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
|
||||||
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
||||||
- vllm_omni/experimental/fullduplex/
|
- vllm_omni/experimental/fullduplex/
|
||||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||||
@@ -210,7 +210,7 @@ steps:
|
|||||||
timeout 20m bash -c "
|
timeout 20m bash -c "
|
||||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
|
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
|
||||||
"
|
"
|
||||||
mirror_hardwares: h100_2
|
mirror_hardwares: h100_1
|
||||||
|
|
||||||
- label: "TTS · VoxCPM2 Test"
|
- label: "TTS · VoxCPM2 Test"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ steps:
|
|||||||
EXIT=$$?
|
EXIT=$$?
|
||||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||||
exit $$EXIT
|
exit $$EXIT
|
||||||
- label: ":robot: Omni · MiniCPM-o 4.5 Test"
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
|
||||||
mirror_hardwares: a3_npu_2
|
mirror_hardwares: a3_npu_2
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
@@ -47,6 +47,34 @@ steps:
|
|||||||
HF_TOKEN: "${HF_TOKEN}"
|
HF_TOKEN: "${HF_TOKEN}"
|
||||||
commands:
|
commands:
|
||||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
||||||
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
mirror_hardwares: a3_npu_2
|
||||||
|
env:
|
||||||
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
|
HF_TOKEN: "${HF_TOKEN}"
|
||||||
|
commands:
|
||||||
|
- >-
|
||||||
|
pytest -s -v
|
||||||
|
tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||||
|
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||||
|
-m "npu and A3 and omni"
|
||||||
|
--run-level full_model
|
||||||
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
mirror_hardwares: a3_npu_2
|
||||||
|
env:
|
||||||
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
|
HF_TOKEN: "${HF_TOKEN}"
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set +e
|
||||||
|
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model
|
||||||
|
EXIT=$$?
|
||||||
|
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||||
|
exit $$EXIT
|
||||||
- group: ":card_index_dividers: TTS Model Test"
|
- group: ":card_index_dividers: TTS Model Test"
|
||||||
key: nightly-tts-test-group
|
key: nightly-tts-test-group
|
||||||
depends_on: upload-nightly-pipeline
|
depends_on: upload-nightly-pipeline
|
||||||
|
|||||||
@@ -3,22 +3,20 @@ steps:
|
|||||||
key: ready-omni-group
|
key: ready-omni-group
|
||||||
depends_on: upload-ready-pipeline
|
depends_on: upload-ready-pipeline
|
||||||
steps:
|
steps:
|
||||||
- label: ":robot: Omni · MiniCPM-o 4.5 Test"
|
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||||
agents:
|
mirror_hardwares: a3_npu_2
|
||||||
queue: "ascend-a3"
|
|
||||||
resource_class: "npu-2"
|
|
||||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
|
||||||
plugins:
|
|
||||||
- kubernetes:
|
|
||||||
podSpecPatch:
|
|
||||||
imagePullSecrets:
|
|
||||||
- name: swr-secret
|
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
HF_TOKEN: "${HF_TOKEN}"
|
|
||||||
commands:
|
commands:
|
||||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||||
|
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||||
|
mirror_hardwares: a3_npu_2
|
||||||
|
env:
|
||||||
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
|
commands:
|
||||||
|
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||||
|
|
||||||
- group: ":card_index_dividers: TTS Model Test"
|
- group: ":card_index_dividers: TTS Model Test"
|
||||||
key: ready-tts-group
|
key: ready-tts-group
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
|||||||
"hyperpyyaml>=1.2.0" \
|
"hyperpyyaml>=1.2.0" \
|
||||||
"onnx>=1.16.0" \
|
"onnx>=1.16.0" \
|
||||||
"librosa>=0.10.0" \
|
"librosa>=0.10.0" \
|
||||||
|
"datasets>=3.0.0" \
|
||||||
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||||
--default-timeout=60 && \
|
--default-timeout=60 && \
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
|||||||
"onnx>=1.16.0" \
|
"onnx>=1.16.0" \
|
||||||
"librosa>=0.10.0" \
|
"librosa>=0.10.0" \
|
||||||
"torchmetrics>=1.9.0" \
|
"torchmetrics>=1.9.0" \
|
||||||
|
"datasets>=3.0.0" \
|
||||||
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||||
--default-timeout=60 && \
|
--default-timeout=60 && \
|
||||||
|
|||||||
@@ -23,10 +23,10 @@ including `librosa`.
|
|||||||
|
|
||||||
The deploy config auto-loads via `--omni`.
|
The deploy config auto-loads via `--omni`.
|
||||||
The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on
|
The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on
|
||||||
logical device 0 with memory budgets of 55%, 15%, and 15%, leaving headroom
|
logical device 0 with memory budgets of 55%, 15%, and 18%, leaving headroom
|
||||||
for runtime kernels such as the HiFi-GAN vocoder's cuDNN workspace. The
|
for runtime kernels such as the HiFi-GAN vocoder's cuDNN workspace. The
|
||||||
profile admits at most four sequences per stage and bounds startup video
|
profile admits at most four sequences per stage and bounds Talker context
|
||||||
profiling to 32 frames per video. For throughput,
|
to 4096 tokens. For throughput,
|
||||||
`minicpmo_4_5_2gpu.yaml` gives the Thinker
|
`minicpmo_4_5_2gpu.yaml` gives the Thinker
|
||||||
GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
|
GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
|
||||||
profile admits at most four concurrent sequences per stage.
|
profile admits at most four concurrent sequences per stage.
|
||||||
|
|||||||
@@ -196,6 +196,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
|||||||
args.model,
|
args.model,
|
||||||
ref_audio=_ref_audio_data_url(args.ref_audio),
|
ref_audio=_ref_audio_data_url(args.ref_audio),
|
||||||
session_id=args.session_id,
|
session_id=args.session_id,
|
||||||
|
temperature=args.temperature,
|
||||||
timeout_s=args.timeout_s,
|
timeout_s=args.timeout_s,
|
||||||
)
|
)
|
||||||
stream_event_cursor = len(client.events.events)
|
stream_event_cursor = len(client.events.events)
|
||||||
@@ -383,6 +384,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
parser.add_argument("--output-dir", default="/tmp/minicpmo_realtime_duplex_demo")
|
parser.add_argument("--output-dir", default="/tmp/minicpmo_realtime_duplex_demo")
|
||||||
parser.add_argument("--chunk-ms", type=int, default=200)
|
parser.add_argument("--chunk-ms", type=int, default=200)
|
||||||
parser.add_argument("--timeout-s", type=float, default=60.0)
|
parser.add_argument("--timeout-s", type=float, default=60.0)
|
||||||
|
parser.add_argument(
|
||||||
|
"--temperature",
|
||||||
|
type=float,
|
||||||
|
default=None,
|
||||||
|
help="Stage0 sampling temperature; response-required soft-interrupt uses 0.0.",
|
||||||
|
)
|
||||||
parser.add_argument("--no-realtime-pacing", action="store_true")
|
parser.add_argument("--no-realtime-pacing", action="store_true")
|
||||||
parser.add_argument("--require-audio", action="store_true")
|
parser.add_argument("--require-audio", action="store_true")
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|||||||
@@ -30,8 +30,9 @@ also provided.
|
|||||||
`hf_config.version="4.5"`):
|
`hf_config.version="4.5"`):
|
||||||
- Default single-GPU compatibility layout (auto-loaded):
|
- Default single-GPU compatibility layout (auto-loaded):
|
||||||
[`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml)
|
[`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml)
|
||||||
- 2-GPU and 3-GPU layouts:
|
- Recommended 2-GPU continuous-batching layout:
|
||||||
[`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml),
|
[`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml),
|
||||||
|
- 3-GPU layout:
|
||||||
[`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml)
|
[`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml)
|
||||||
- 8x RTX 4090 layout:
|
- 8x RTX 4090 layout:
|
||||||
[`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml)
|
[`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml)
|
||||||
|
|||||||
@@ -6,13 +6,21 @@ Unit tests for patch.py
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import base64
|
||||||
import json
|
import json
|
||||||
|
import time
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from pytest_mock import MockerFixture
|
from pytest_mock import MockerFixture
|
||||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||||
|
|
||||||
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput, async_request_openai_chat_omni_completions
|
from vllm_omni.benchmarks.patch.patch import (
|
||||||
|
MixRequestFuncOutput,
|
||||||
|
async_request_openai_chat_omni_completions,
|
||||||
|
async_request_openai_realtime_duplex,
|
||||||
|
)
|
||||||
|
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
||||||
|
|
||||||
@@ -40,6 +48,156 @@ class MockResponse:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch):
|
||||||
|
class FakeRealtimeClient:
|
||||||
|
last_instance = None
|
||||||
|
|
||||||
|
def __init__(self, url):
|
||||||
|
assert url == "ws://localhost:8000/v1/realtime?duplex=1"
|
||||||
|
self.events = RealtimeEventCollector()
|
||||||
|
self.configure_kwargs = None
|
||||||
|
self.sent = []
|
||||||
|
self.response_count = 0
|
||||||
|
self.ack_count = 0
|
||||||
|
type(self).last_instance = self
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *_args):
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def configure(self, model, **kwargs):
|
||||||
|
assert model == "openbmb/MiniCPM-o-4_5"
|
||||||
|
self.configure_kwargs = kwargs
|
||||||
|
|
||||||
|
async def send(self, event):
|
||||||
|
self.sent.append(event)
|
||||||
|
if event["type"] != "response.create":
|
||||||
|
return
|
||||||
|
self.response_count += 1
|
||||||
|
response_id = f"resp-{self.response_count}"
|
||||||
|
now = time.monotonic()
|
||||||
|
self.events.add(
|
||||||
|
{"type": "response.created", "response": {"id": response_id}},
|
||||||
|
received_at_s=now + 0.01,
|
||||||
|
)
|
||||||
|
self.events.add(
|
||||||
|
{
|
||||||
|
"type": "response.audio_transcript.delta",
|
||||||
|
"response_id": response_id,
|
||||||
|
"delta": f"turn {self.response_count}",
|
||||||
|
},
|
||||||
|
received_at_s=now + 0.02,
|
||||||
|
)
|
||||||
|
self.events.add(
|
||||||
|
{
|
||||||
|
"type": "response.audio.delta",
|
||||||
|
"response_id": response_id,
|
||||||
|
"delta": base64.b64encode(b"\x00\x00" * 2400).decode(),
|
||||||
|
"sample_rate_hz": 24_000,
|
||||||
|
"metadata": {"audio_duration_ms": 100},
|
||||||
|
},
|
||||||
|
received_at_s=now + 0.03,
|
||||||
|
)
|
||||||
|
self.events.add(
|
||||||
|
{"type": "response.done", "response": {"id": response_id}},
|
||||||
|
received_at_s=now + 0.04,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def acknowledge_playback(self):
|
||||||
|
self.ack_count += 1
|
||||||
|
|
||||||
|
async def close_session(self, **_kwargs):
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm_omni.benchmarks.patch.patch.RealtimeDuplexClient",
|
||||||
|
FakeRealtimeClient,
|
||||||
|
)
|
||||||
|
request_input = RequestFuncInput(
|
||||||
|
model="openbmb/MiniCPM-o-4_5",
|
||||||
|
model_name="openbmb/MiniCPM-o-4_5",
|
||||||
|
prompt="hello",
|
||||||
|
api_url="http://localhost:8000/v1/realtime",
|
||||||
|
prompt_len=1,
|
||||||
|
output_len=20,
|
||||||
|
logprobs=None,
|
||||||
|
multi_modal_content=None,
|
||||||
|
ignore_eos=False,
|
||||||
|
extra_body={"save_duplex_request_metrics": True},
|
||||||
|
)
|
||||||
|
request_input.seed_tts_speech_extra = {"ref_audio": "data:audio/wav;base64,AAAA"}
|
||||||
|
request_input.seed_tts_system_prompt = "Speak exactly."
|
||||||
|
request_input.seed_tts_turns = tuple(
|
||||||
|
SimpleNamespace(utterance_id=f"utt-{index}", target_text=f"text {index}") for index in range(4)
|
||||||
|
)
|
||||||
|
|
||||||
|
output = await async_request_openai_realtime_duplex(
|
||||||
|
request_input,
|
||||||
|
session=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
client = FakeRealtimeClient.last_instance
|
||||||
|
assert client.configure_kwargs["native_duplex"] is False
|
||||||
|
assert client.configure_kwargs["extra_body"] == {
|
||||||
|
"ref_audio": "data:audio/wav;base64,AAAA",
|
||||||
|
}
|
||||||
|
assert client.sent == [
|
||||||
|
event
|
||||||
|
for index in range(4)
|
||||||
|
for event in (
|
||||||
|
{
|
||||||
|
"type": "conversation.item.create",
|
||||||
|
"item": {
|
||||||
|
"type": "message",
|
||||||
|
"role": "user",
|
||||||
|
"content": [{"type": "input_text", "text": f"text {index}"}],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{"type": "response.create"},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
assert client.ack_count == 4
|
||||||
|
assert output.success is True
|
||||||
|
assert output.generated_text == "turn 1 turn 2 turn 3 turn 4"
|
||||||
|
assert output.audio_duration == pytest.approx(0.4)
|
||||||
|
assert output.ttft > 0
|
||||||
|
assert output.audio_ttfp > output.ttft
|
||||||
|
assert output.audio_rtf > 0
|
||||||
|
assert output.latency > 0
|
||||||
|
assert output.tts_turn_pcm_bytes == [b"\x00\x00" * 2400] * 4
|
||||||
|
assert output.tts_output_pcm_bytes == b"\x00\x00" * 9600
|
||||||
|
session_id = output.duplex_request_metrics[0]["session_id"]
|
||||||
|
assert len(output.duplex_request_metrics) == 4
|
||||||
|
for index, metrics in enumerate(output.duplex_request_metrics):
|
||||||
|
assert metrics == {
|
||||||
|
"session_id": session_id,
|
||||||
|
"request_index": index,
|
||||||
|
"utterance_id": f"utt-{index}",
|
||||||
|
"response_id": f"resp-{index + 1}",
|
||||||
|
"source": "client_monotonic_receive",
|
||||||
|
"measurement_origin": {
|
||||||
|
"ttft": "conversation.item.create client send to first non-empty text delta",
|
||||||
|
"ttfp": "conversation.item.create client send to first audio packet",
|
||||||
|
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
|
||||||
|
},
|
||||||
|
"ttft_ms": pytest.approx(20.0, abs=2.0),
|
||||||
|
"ttfp_ms": pytest.approx(30.0, abs=2.0),
|
||||||
|
"rtf": pytest.approx(0.3, abs=0.03),
|
||||||
|
"audio_generation_ms": pytest.approx(30.0, abs=2.0),
|
||||||
|
"audio_duration_ms": 100.0,
|
||||||
|
}
|
||||||
|
assert output.duplex_session_metrics == {
|
||||||
|
"session_id": session_id,
|
||||||
|
"audio_turn_count": 4,
|
||||||
|
"mean_ttft_ms": pytest.approx(20.0, abs=2.0),
|
||||||
|
"mean_ttfp_ms": pytest.approx(30.0, abs=2.0),
|
||||||
|
"mean_rtf": pytest.approx(0.3, abs=0.03),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def create_sse_chunk(data_dict):
|
def create_sse_chunk(data_dict):
|
||||||
"""Helper to create SSE formatted chunk"""
|
"""Helper to create SSE formatted chunk"""
|
||||||
return f"data: {json.dumps(data_dict)}\n\n".encode()
|
return f"data: {json.dumps(data_dict)}\n\n".encode()
|
||||||
|
|||||||
@@ -42,7 +42,10 @@ from benchmarks.accuracy.text_to_image.gbench import (
|
|||||||
summarize_generated_records as summarize_gebench_generated_records,
|
summarize_generated_records as summarize_gebench_generated_records,
|
||||||
summarize_gebench_results,
|
summarize_gebench_results,
|
||||||
)
|
)
|
||||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import seed_tts_bench_argv
|
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||||
|
build_serve_common_argv,
|
||||||
|
seed_tts_bench_argv,
|
||||||
|
)
|
||||||
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
|
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
|
||||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
|
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
|
||||||
|
|
||||||
@@ -57,6 +60,25 @@ def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
|
|||||||
assert argv[dataset_idx + 1] == "zhaochenyang20/seed-tts-eval"
|
assert argv[dataset_idx + 1] == "zhaochenyang20/seed-tts-eval"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_serve_common_argv_accepts_realtime_seed_tts_route(tmp_path):
|
||||||
|
argv = build_serve_common_argv(
|
||||||
|
host="127.0.0.1",
|
||||||
|
port=8000,
|
||||||
|
model="openbmb/MiniCPM-o-4_5",
|
||||||
|
num_prompts=4,
|
||||||
|
max_concurrency=1,
|
||||||
|
num_warmups=0,
|
||||||
|
percentile_metrics="ttft,audio_ttfp,audio_rtf",
|
||||||
|
result_dir=tmp_path,
|
||||||
|
result_filename="result.json",
|
||||||
|
backend="openai-realtime-tts",
|
||||||
|
endpoint="/v1/realtime",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert argv[argv.index("--backend") + 1] == "openai-realtime-tts"
|
||||||
|
assert argv[argv.index("--endpoint") + 1] == "/v1/realtime"
|
||||||
|
|
||||||
|
|
||||||
def test_sync_dataset_env_preserves_seed_tts_hf_repo_id(monkeypatch):
|
def test_sync_dataset_env_preserves_seed_tts_hf_repo_id(monkeypatch):
|
||||||
ns = argparse.Namespace(
|
ns = argparse.Namespace(
|
||||||
daily_omni_repo=None,
|
daily_omni_repo=None,
|
||||||
|
|||||||
@@ -28,10 +28,13 @@ if _MODULE_NAME not in sys.modules:
|
|||||||
_spec.loader.exec_module(_mod)
|
_spec.loader.exec_module(_mod)
|
||||||
|
|
||||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( # noqa: E402
|
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( # noqa: E402
|
||||||
|
SeedTTSDataset,
|
||||||
SeedTTSDesignDataset,
|
SeedTTSDesignDataset,
|
||||||
SeedTTSDesignSampleRequest,
|
SeedTTSDesignSampleRequest,
|
||||||
|
SeedTTSSampleRequest,
|
||||||
SeedTTSTextDataset,
|
SeedTTSTextDataset,
|
||||||
SeedTTSTextSampleRequest,
|
SeedTTSTextSampleRequest,
|
||||||
|
SeedTTSTurn,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -70,6 +73,94 @@ def mock_tokenizer(mocker):
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_seed_tts_dataset_groups_four_turns_with_one_reference(seed_tts_root, mock_tokenizer):
|
||||||
|
ds = SeedTTSDataset(
|
||||||
|
dataset_path=str(seed_tts_root),
|
||||||
|
random_seed=0,
|
||||||
|
locale="en",
|
||||||
|
disable_shuffle=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
requests = ds.sample(
|
||||||
|
mock_tokenizer,
|
||||||
|
num_requests=1,
|
||||||
|
turns_per_session=4,
|
||||||
|
no_oversample=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(requests) == 1
|
||||||
|
request = requests[0]
|
||||||
|
assert [turn.utterance_id for turn in request.seed_tts_turns] == [
|
||||||
|
"utt000",
|
||||||
|
"utt001",
|
||||||
|
"utt002",
|
||||||
|
"utt003",
|
||||||
|
]
|
||||||
|
assert [turn.target_text for turn in request.seed_tts_turns] == [
|
||||||
|
"target text 0",
|
||||||
|
"target text 1",
|
||||||
|
"target text 2",
|
||||||
|
"target text 3",
|
||||||
|
]
|
||||||
|
assert request.seed_tts_speech_extra["ref_text"] == "ref text 0"
|
||||||
|
assert request.seed_tts_ref_wav_path.endswith("utt000.wav")
|
||||||
|
|
||||||
|
|
||||||
|
def test_seed_tts_eval_expands_grouped_pcm_by_turn():
|
||||||
|
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||||
|
_expand_seed_tts_turn_outputs,
|
||||||
|
)
|
||||||
|
|
||||||
|
request = SeedTTSSampleRequest(
|
||||||
|
prompt="target 0",
|
||||||
|
prompt_len=4,
|
||||||
|
expected_output_len=100,
|
||||||
|
multi_modal_data=None,
|
||||||
|
request_id="session-0",
|
||||||
|
seed_tts_turns=tuple(
|
||||||
|
SeedTTSTurn(utterance_id=f"utt-{index}", target_text=f"target {index}") for index in range(4)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
output = types.SimpleNamespace(
|
||||||
|
success=True,
|
||||||
|
tts_turn_pcm_bytes=[bytes([index]) for index in range(4)],
|
||||||
|
)
|
||||||
|
|
||||||
|
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
|
||||||
|
|
||||||
|
assert [item.prompt for item in requests] == [f"target {index}" for index in range(4)]
|
||||||
|
assert [item.seed_tts_utterance_id for item in requests] == [f"utt-{index}" for index in range(4)]
|
||||||
|
assert [item.tts_output_pcm_bytes for item in outputs] == [bytes([index]) for index in range(4)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_seed_tts_eval_keeps_session_pcm_for_single_turn_chat():
|
||||||
|
"""Chat-omni Seed-TTS sets session PCM only; expand must not wipe it."""
|
||||||
|
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||||
|
_expand_seed_tts_turn_outputs,
|
||||||
|
)
|
||||||
|
|
||||||
|
request = SeedTTSSampleRequest(
|
||||||
|
prompt="target 0",
|
||||||
|
prompt_len=4,
|
||||||
|
expected_output_len=100,
|
||||||
|
multi_modal_data=None,
|
||||||
|
request_id="session-0",
|
||||||
|
seed_tts_turns=(SeedTTSTurn(utterance_id="utt-0", target_text="target 0"),),
|
||||||
|
)
|
||||||
|
pcm = b"\x01\x02\x03\x04"
|
||||||
|
output = types.SimpleNamespace(
|
||||||
|
success=True,
|
||||||
|
tts_output_pcm_bytes=pcm,
|
||||||
|
tts_turn_pcm_bytes=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
|
||||||
|
|
||||||
|
assert len(requests) == 1
|
||||||
|
assert requests[0].prompt == "target 0"
|
||||||
|
assert outputs[0].tts_output_pcm_bytes == pcm
|
||||||
|
|
||||||
|
|
||||||
def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mock_tokenizer):
|
def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mock_tokenizer):
|
||||||
ds = SeedTTSTextDataset(
|
ds = SeedTTSTextDataset(
|
||||||
dataset_path=str(seed_tts_root),
|
dataset_path=str(seed_tts_root),
|
||||||
|
|||||||
@@ -2241,7 +2241,8 @@ class TestPlatformOverrides:
|
|||||||
load_deploy_config(Path(get_deploy_config_path(filename))), platform="cuda"
|
load_deploy_config(Path(get_deploy_config_path(filename))), platform="cuda"
|
||||||
)
|
)
|
||||||
replica_stages = merge_pipeline_deploy(pipeline, replica)
|
replica_stages = merge_pipeline_deploy(pipeline, replica)
|
||||||
assert replica_stages[1].yaml_engine_args["kv_cache_memory_bytes"] is None
|
# Explicit null clears the inherited single-GPU 2 GiB CUDA cap.
|
||||||
|
assert replica_stages[1].yaml_engine_args.get("kv_cache_memory_bytes") is None
|
||||||
|
|
||||||
def test_npu_overrides(self):
|
def test_npu_overrides(self):
|
||||||
deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml"))
|
deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml"))
|
||||||
|
|||||||
@@ -474,6 +474,8 @@ def test_sub_config_fields_match_structured_scopes():
|
|||||||
"moe_backend",
|
"moe_backend",
|
||||||
"hf_overrides",
|
"hf_overrides",
|
||||||
"limit_mm_per_prompt",
|
"limit_mm_per_prompt",
|
||||||
|
"interleave_mm_strings",
|
||||||
|
"media_io_kwargs",
|
||||||
"active_stream_window",
|
"active_stream_window",
|
||||||
"duplex_max_sessions",
|
"duplex_max_sessions",
|
||||||
"enable_sleep_mode",
|
"enable_sleep_mode",
|
||||||
|
|||||||
@@ -752,3 +752,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
|
|||||||
default=None,
|
default=None,
|
||||||
help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
|
help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
|
||||||
)
|
)
|
||||||
|
parser.addoption(
|
||||||
|
"--assert-baseline",
|
||||||
|
action="store_true",
|
||||||
|
default=False,
|
||||||
|
help=(
|
||||||
|
"When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
|
||||||
|
"(default: off)."
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -157,8 +157,19 @@ def benchmark_params(request):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def assert_result(result, num_prompt) -> None:
|
def assert_result(result, params, num_prompt) -> None:
|
||||||
assert result["completed"] == num_prompt, "Request failures exist"
|
assert result["completed"] == num_prompt, "Request failures exist"
|
||||||
|
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
|
||||||
|
if expected_audio_turns is not None:
|
||||||
|
session_metrics = result.get("duplex_session_metrics")
|
||||||
|
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
|
||||||
|
assert len(session_metrics) == num_prompt, (
|
||||||
|
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
|
||||||
|
)
|
||||||
|
assert all(
|
||||||
|
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
|
||||||
|
for metric in session_metrics
|
||||||
|
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.benchmark
|
@pytest.mark.benchmark
|
||||||
@@ -212,6 +223,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
|||||||
"enabled",
|
"enabled",
|
||||||
"eval_phase",
|
"eval_phase",
|
||||||
"trust_remote_code",
|
"trust_remote_code",
|
||||||
|
"expected_duplex_audio_turns_per_session",
|
||||||
}
|
}
|
||||||
|
|
||||||
for key, value in params.items():
|
for key, value in params.items():
|
||||||
@@ -251,7 +263,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
|||||||
random_output_len=params.get("random_output_len"),
|
random_output_len=params.get("random_output_len"),
|
||||||
resource_label=resource_label,
|
resource_label=resource_label,
|
||||||
)
|
)
|
||||||
assert_result(result, num_prompt)
|
assert_result(result, params, num_prompt)
|
||||||
|
|
||||||
# concurrency test
|
# concurrency test
|
||||||
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||||
@@ -268,4 +280,4 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
|||||||
random_output_len=params.get("random_output_len"),
|
random_output_len=params.get("random_output_len"),
|
||||||
resource_label=resource_label,
|
resource_label=resource_label,
|
||||||
)
|
)
|
||||||
assert_result(result, num_prompt)
|
assert_result(result, params, num_prompt)
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"test_name": "test_minicpmo_4_5",
|
||||||
|
"mark": [
|
||||||
|
{
|
||||||
|
"hardware_marks": {
|
||||||
|
"res": {
|
||||||
|
"cuda": "H100",
|
||||||
|
"npu": "A3"
|
||||||
|
},
|
||||||
|
"num_cards": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"full_model",
|
||||||
|
"omni"
|
||||||
|
],
|
||||||
|
"server_params": {
|
||||||
|
"model": "openbmb/MiniCPM-o-4_5",
|
||||||
|
"extra_cli_args": [
|
||||||
|
"--deploy-config",
|
||||||
|
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||||
|
"--trust-remote-code"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"benchmark_params": [
|
||||||
|
{
|
||||||
|
"dataset_name": "seed-tts",
|
||||||
|
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||||
|
"backend": "openai-chat-omni",
|
||||||
|
"endpoint": "/v1/chat/completions",
|
||||||
|
"num_prompts": [
|
||||||
|
32,
|
||||||
|
64,
|
||||||
|
128
|
||||||
|
],
|
||||||
|
"max_concurrency": [
|
||||||
|
1,
|
||||||
|
4,
|
||||||
|
8
|
||||||
|
],
|
||||||
|
"no_oversample": true,
|
||||||
|
"trust_remote_code": true,
|
||||||
|
"seed_tts_locale": "en",
|
||||||
|
"extra_body": {
|
||||||
|
"modalities": [
|
||||||
|
"text",
|
||||||
|
"audio"
|
||||||
|
],
|
||||||
|
"chat_template_kwargs": {
|
||||||
|
"enable_thinking": false,
|
||||||
|
"use_tts_template": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||||
|
"baseline": {
|
||||||
|
"H100": {
|
||||||
|
"request_throughput": [
|
||||||
|
0.7803,
|
||||||
|
1.1008,
|
||||||
|
1.2027
|
||||||
|
],
|
||||||
|
"mean_ttft_ms": [
|
||||||
|
166.9764,
|
||||||
|
307.7133,
|
||||||
|
323.584
|
||||||
|
],
|
||||||
|
"mean_e2el_ms": [
|
||||||
|
1361.1682,
|
||||||
|
3800.6332,
|
||||||
|
6961.8636
|
||||||
|
],
|
||||||
|
"mean_audio_ttfp_ms": [
|
||||||
|
635.8099,
|
||||||
|
1615.6591,
|
||||||
|
2888.9831
|
||||||
|
],
|
||||||
|
"mean_audio_rtf": [
|
||||||
|
0.3151,
|
||||||
|
0.8568,
|
||||||
|
1.5425
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"test_name": "test_minicpmo_4_5_duplex_seed_tts",
|
||||||
|
"mark": [
|
||||||
|
{
|
||||||
|
"hardware_marks": {
|
||||||
|
"res": {
|
||||||
|
"cuda": "H100",
|
||||||
|
"npu": "A3"
|
||||||
|
},
|
||||||
|
"num_cards": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"full_model",
|
||||||
|
"omni"
|
||||||
|
],
|
||||||
|
"server_params": {
|
||||||
|
"model": "openbmb/MiniCPM-o-4_5",
|
||||||
|
"extra_cli_args": [
|
||||||
|
"--deploy-config",
|
||||||
|
"vllm_omni/deploy/minicpmo_4_5_duplex.yaml",
|
||||||
|
"--trust-remote-code"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"benchmark_params": [
|
||||||
|
{
|
||||||
|
"dataset_name": "seed-tts",
|
||||||
|
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||||
|
"backend": "openai-realtime-tts",
|
||||||
|
"endpoint": "/v1/realtime",
|
||||||
|
"num_prompts": [
|
||||||
|
1
|
||||||
|
],
|
||||||
|
"max_concurrency": [
|
||||||
|
1
|
||||||
|
],
|
||||||
|
"no_oversample": true,
|
||||||
|
"trust_remote_code": true,
|
||||||
|
"seed_tts_locale": "en",
|
||||||
|
"seed_tts_turns_per_session": 4,
|
||||||
|
"expected_duplex_audio_turns_per_session": 4,
|
||||||
|
"extra_body": {
|
||||||
|
"save_duplex_request_metrics": true
|
||||||
|
},
|
||||||
|
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||||
|
"baseline": {
|
||||||
|
"H100": {
|
||||||
|
"request_throughput": [
|
||||||
|
0.1594
|
||||||
|
],
|
||||||
|
"mean_ttft_ms": [
|
||||||
|
274.164
|
||||||
|
],
|
||||||
|
"mean_e2el_ms": [
|
||||||
|
6250.7696
|
||||||
|
],
|
||||||
|
"mean_audio_ttfp_ms": [
|
||||||
|
994.422
|
||||||
|
],
|
||||||
|
"mean_audio_rtf": [
|
||||||
|
0.46163
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -485,3 +485,30 @@ def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkey
|
|||||||
assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3
|
assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3
|
||||||
assert "A3" in sweeps[2]["params"]["baseline"]
|
assert "A3" in sweeps[2]["params"]["baseline"]
|
||||||
assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list)
|
assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list)
|
||||||
|
|
||||||
|
|
||||||
|
def test_omni_duplex_expected_audio_turns_accepts_complete_session():
|
||||||
|
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||||
|
|
||||||
|
assert_result(
|
||||||
|
{
|
||||||
|
"completed": 1,
|
||||||
|
"duplex_session_metrics": [{"audio_turn_count": 4}],
|
||||||
|
},
|
||||||
|
{"expected_duplex_audio_turns_per_session": 4},
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_omni_duplex_expected_audio_turns_rejects_incomplete_session():
|
||||||
|
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||||
|
|
||||||
|
with pytest.raises(AssertionError, match="emitted 4 audio turns"):
|
||||||
|
assert_result(
|
||||||
|
{
|
||||||
|
"completed": 1,
|
||||||
|
"duplex_session_metrics": [{"audio_turn_count": 3}],
|
||||||
|
},
|
||||||
|
{"expected_duplex_audio_turns_per_session": 4},
|
||||||
|
1,
|
||||||
|
)
|
||||||
@@ -50,12 +50,12 @@ def _prompt() -> str:
|
|||||||
return "What is the capital of China? Answer in 20 words."
|
return "What is the capital of China? Answer in 20 words."
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True)
|
@pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True)
|
||||||
def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None:
|
def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None:
|
||||||
"""
|
"""
|
||||||
Test that invalid audio format is properly rejected.
|
Test that invalid audio format is properly rejected.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text + invalid audio format
|
Input Modal: text + invalid audio format
|
||||||
Output Modal: error
|
Output Modal: error
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
|
||||||
|
|
||||||
|
Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
|
||||||
|
~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
|
||||||
|
text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
|
||||||
|
+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.e2e.accuracy.qwen3_omni import run_qwen_omni_acc_benchmark as _acc_bench
|
||||||
|
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||||
|
build_acc_benchmark_cli_argv,
|
||||||
|
find_vllm_cli,
|
||||||
|
)
|
||||||
|
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import SERVER_PARAMS as DUPLEX_TEST_PARAMS
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
|
from tests.helpers.runtime import OmniServerParams
|
||||||
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
|
|
||||||
|
_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5")
|
||||||
|
_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml")
|
||||||
|
_RESULT_DIR = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"ACC_BENCH_RESULT_DIR",
|
||||||
|
str(Path(__file__).resolve().parent / "results"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
_MIN_DAILY_OMNI_ACCURACY = 0.78
|
||||||
|
_MAX_SEED_TTS_MEAN_WER = 0.05
|
||||||
|
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
|
||||||
|
_DAILY_EXTRA_BODY = {
|
||||||
|
"modalities": ["text"],
|
||||||
|
"chat_template_kwargs": {
|
||||||
|
"enable_thinking": False,
|
||||||
|
"use_tts_template": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_SEED_EXTRA_BODY = {
|
||||||
|
"modalities": ["text", "audio"],
|
||||||
|
"chat_template_kwargs": {
|
||||||
|
"enable_thinking": False,
|
||||||
|
"use_tts_template": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
# Server flags required for MiniCPM interleaved image/audio packs (Daily-Omni only).
|
||||||
|
# Do not reuse these for Seed-TTS: ``--interleave-mm-strings`` + TTS ref_audio can trip
|
||||||
|
# msgspec ValidationError and kill the orchestrator mid-suite.
|
||||||
|
# Also pinned in ``minicpmo_4_5.yaml`` stage 0 so they survive config-path routing.
|
||||||
|
_DAILY_OMNI_SERVER_ARGS = [
|
||||||
|
"--trust-remote-code",
|
||||||
|
"--interleave-mm-strings",
|
||||||
|
"--media-io-kwargs",
|
||||||
|
'{"video":{"fps":1,"num_frames":128}}',
|
||||||
|
]
|
||||||
|
_SEED_TTS_SERVER_ARGS = [
|
||||||
|
"--trust-remote-code",
|
||||||
|
]
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||||
|
|
||||||
|
# Evaluated at setup time, before the ``omni_server`` fixture boots a server. An
|
||||||
|
# ``importorskip`` inside the test body would skip only after paying several minutes
|
||||||
|
# of startup, and the resulting skip is easy to miss in a nightly log.
|
||||||
|
_missing_daily_omni_deps = [name for name in ("datasets", "huggingface_hub") if importlib.util.find_spec(name) is None]
|
||||||
|
requires_daily_omni_deps = pytest.mark.skipif(
|
||||||
|
bool(_missing_daily_omni_deps),
|
||||||
|
reason=f"Daily-Omni accuracy bench needs {', '.join(_missing_daily_omni_deps)}",
|
||||||
|
)
|
||||||
|
|
||||||
|
daily_test_params = [
|
||||||
|
OmniServerParams(
|
||||||
|
model=_MODEL,
|
||||||
|
stage_config_path=_DEPLOY_CONFIG,
|
||||||
|
server_args=list(_DAILY_OMNI_SERVER_ARGS),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
seed_test_params = [
|
||||||
|
OmniServerParams(
|
||||||
|
model=_MODEL,
|
||||||
|
stage_config_path=_DEPLOY_CONFIG,
|
||||||
|
server_args=list(_SEED_TTS_SERVER_ARGS),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _require_vllm_cli() -> None:
|
||||||
|
try:
|
||||||
|
find_vllm_cli()
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
pytest.skip(str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""Inline cached Hub/local videos because the test server has no local-media allowlist."""
|
||||||
|
original = _acc_bench.daily_omni_bench_argv
|
||||||
|
|
||||||
|
def _wrapped() -> list[str]:
|
||||||
|
argv = list(original())
|
||||||
|
if "--daily-omni-inline-local-video" not in argv:
|
||||||
|
argv.append("--daily-omni-inline-local-video")
|
||||||
|
return argv
|
||||||
|
|
||||||
|
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
|
||||||
|
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||||
|
|
||||||
|
|
||||||
|
@requires_daily_omni_deps
|
||||||
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
|
@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
|
||||||
|
def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
|
||||||
|
_require_vllm_cli()
|
||||||
|
|
||||||
|
argv = build_acc_benchmark_cli_argv(
|
||||||
|
omni_server,
|
||||||
|
skip_seed=True,
|
||||||
|
skip_daily=False,
|
||||||
|
)
|
||||||
|
argv.extend(
|
||||||
|
[
|
||||||
|
"--result-dir",
|
||||||
|
str(_RESULT_DIR),
|
||||||
|
"--min-daily-omni-accuracy",
|
||||||
|
str(_MIN_DAILY_OMNI_ACCURACY),
|
||||||
|
"--temperature",
|
||||||
|
"0",
|
||||||
|
"--output-len",
|
||||||
|
"512",
|
||||||
|
"--daily-omni-input-mode",
|
||||||
|
"all",
|
||||||
|
"--daily-omni-pack-mode",
|
||||||
|
"minicpm-interleave",
|
||||||
|
"--daily-extra-body-json",
|
||||||
|
json.dumps(_DAILY_EXTRA_BODY, separators=(",", ":")),
|
||||||
|
"--trust-remote-code",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
|
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
|
||||||
|
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||||
|
_require_vllm_cli()
|
||||||
|
pytest.importorskip("huggingface_hub")
|
||||||
|
|
||||||
|
argv = build_acc_benchmark_cli_argv(
|
||||||
|
omni_server,
|
||||||
|
skip_seed=False,
|
||||||
|
skip_daily=True,
|
||||||
|
)
|
||||||
|
argv.extend(
|
||||||
|
[
|
||||||
|
"--result-dir",
|
||||||
|
str(_RESULT_DIR),
|
||||||
|
"--max-seed-tts-mean-wer",
|
||||||
|
str(_MAX_SEED_TTS_MEAN_WER),
|
||||||
|
"--seed-extra-body-json",
|
||||||
|
json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")),
|
||||||
|
"--trust-remote-code",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
@pytest.mark.parametrize("omni_server", DUPLEX_TEST_PARAMS, indirect=True)
|
||||||
|
def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
|
||||||
|
"""Gate Seed-TTS WER through the explicit Realtime TTS contract."""
|
||||||
|
_require_vllm_cli()
|
||||||
|
pytest.importorskip("huggingface_hub")
|
||||||
|
|
||||||
|
argv = build_acc_benchmark_cli_argv(
|
||||||
|
omni_server,
|
||||||
|
skip_seed=False,
|
||||||
|
skip_daily=True,
|
||||||
|
num_prompts=50,
|
||||||
|
max_concurrency=1,
|
||||||
|
)
|
||||||
|
argv.extend(
|
||||||
|
[
|
||||||
|
"--result-dir",
|
||||||
|
str(_RESULT_DIR),
|
||||||
|
"--max-seed-tts-mean-wer",
|
||||||
|
str(_MAX_SEED_TTS_MEAN_WER),
|
||||||
|
"--seed-extra-body-json",
|
||||||
|
json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")),
|
||||||
|
"--seed-backend",
|
||||||
|
"openai-realtime-tts",
|
||||||
|
"--seed-endpoint",
|
||||||
|
"/v1/realtime",
|
||||||
|
"--seed-tts-turns-per-session",
|
||||||
|
"4",
|
||||||
|
"--trust-remote-code",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||||
@@ -19,6 +19,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Protocol
|
from typing import Any, Protocol
|
||||||
|
|
||||||
@@ -121,9 +122,13 @@ def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
|
|||||||
|
|
||||||
def find_vllm_cli() -> str:
|
def find_vllm_cli() -> str:
|
||||||
exe = shutil.which("vllm")
|
exe = shutil.which("vllm")
|
||||||
if not exe:
|
if exe:
|
||||||
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
|
return exe
|
||||||
return exe
|
# Fallback when pytest is invoked via absolute venv path without activating PATH.
|
||||||
|
sibling = Path(sys.executable).resolve().parent / "vllm"
|
||||||
|
if sibling.is_file() and os.access(sibling, os.X_OK):
|
||||||
|
return str(sibling)
|
||||||
|
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
|
||||||
|
|
||||||
|
|
||||||
def run_vllm_bench_subprocess(vllm: str, argv: list[str], *, extra_env: dict[str, str] | None = None) -> None:
|
def run_vllm_bench_subprocess(vllm: str, argv: list[str], *, extra_env: dict[str, str] | None = None) -> None:
|
||||||
@@ -150,6 +155,11 @@ def build_serve_common_argv(
|
|||||||
result_dir: Path,
|
result_dir: Path,
|
||||||
result_filename: str,
|
result_filename: str,
|
||||||
ready_check_timeout_sec: int | None = None,
|
ready_check_timeout_sec: int | None = None,
|
||||||
|
trust_remote_code: bool = False,
|
||||||
|
backend: str = "openai-chat-omni",
|
||||||
|
endpoint: str = "/v1/chat/completions",
|
||||||
|
temperature: float | None = None,
|
||||||
|
output_len: int | None = None,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
out = [
|
out = [
|
||||||
"bench",
|
"bench",
|
||||||
@@ -162,9 +172,9 @@ def build_serve_common_argv(
|
|||||||
"--model",
|
"--model",
|
||||||
model,
|
model,
|
||||||
"--endpoint",
|
"--endpoint",
|
||||||
"/v1/chat/completions",
|
endpoint,
|
||||||
"--backend",
|
"--backend",
|
||||||
"openai-chat-omni",
|
backend,
|
||||||
"--request-rate",
|
"--request-rate",
|
||||||
"inf",
|
"inf",
|
||||||
"--num-prompts",
|
"--num-prompts",
|
||||||
@@ -184,6 +194,12 @@ def build_serve_common_argv(
|
|||||||
]
|
]
|
||||||
if ready_check_timeout_sec is not None:
|
if ready_check_timeout_sec is not None:
|
||||||
out.extend(["--ready-check-timeout-sec", str(int(ready_check_timeout_sec))])
|
out.extend(["--ready-check-timeout-sec", str(int(ready_check_timeout_sec))])
|
||||||
|
if trust_remote_code:
|
||||||
|
out.append("--trust-remote-code")
|
||||||
|
if temperature is not None:
|
||||||
|
out.extend(["--temperature", str(temperature)])
|
||||||
|
if output_len is not None:
|
||||||
|
out.extend(["--output-len", str(int(output_len))])
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -171,7 +171,13 @@ def _preserve_benchmark_dataset_env() -> Any:
|
|||||||
os.environ[key] = value
|
os.environ[key] = value
|
||||||
|
|
||||||
|
|
||||||
def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[str]:
|
def _build_common_args(
|
||||||
|
ns: argparse.Namespace,
|
||||||
|
*,
|
||||||
|
result_filename: str,
|
||||||
|
backend: str = "openai-chat-omni",
|
||||||
|
endpoint: str = "/v1/chat/completions",
|
||||||
|
) -> list[str]:
|
||||||
return build_serve_common_argv(
|
return build_serve_common_argv(
|
||||||
host=ns.host,
|
host=ns.host,
|
||||||
port=ns.port,
|
port=ns.port,
|
||||||
@@ -183,6 +189,11 @@ def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[
|
|||||||
result_dir=ns.result_dir,
|
result_dir=ns.result_dir,
|
||||||
result_filename=result_filename,
|
result_filename=result_filename,
|
||||||
ready_check_timeout_sec=ns.ready_check_timeout_sec,
|
ready_check_timeout_sec=ns.ready_check_timeout_sec,
|
||||||
|
trust_remote_code=ns.trust_remote_code,
|
||||||
|
backend=backend,
|
||||||
|
endpoint=endpoint,
|
||||||
|
temperature=getattr(ns, "temperature", None),
|
||||||
|
output_len=getattr(ns, "output_len", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -201,6 +212,9 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
|
|||||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
pack_mode = getattr(ns, "daily_omni_pack_mode", None)
|
||||||
|
if pack_mode:
|
||||||
|
argv.extend(["--daily-omni-pack-mode", pack_mode])
|
||||||
if ns.daily_omni_save_eval_items:
|
if ns.daily_omni_save_eval_items:
|
||||||
argv.append("--daily-omni-save-eval-items")
|
argv.append("--daily-omni-save-eval-items")
|
||||||
print("\n$", vllm, *argv, "\n", flush=True)
|
print("\n$", vllm, *argv, "\n", flush=True)
|
||||||
@@ -217,7 +231,12 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
|||||||
result_filename = f"qwen_omni_acc_seed_tts_{tag}.json"
|
result_filename = f"qwen_omni_acc_seed_tts_{tag}.json"
|
||||||
extra = json.loads(ns.seed_extra_body_json)
|
extra = json.loads(ns.seed_extra_body_json)
|
||||||
argv = (
|
argv = (
|
||||||
_build_common_args(ns, result_filename=result_filename)
|
_build_common_args(
|
||||||
|
ns,
|
||||||
|
result_filename=result_filename,
|
||||||
|
backend=ns.seed_backend,
|
||||||
|
endpoint=ns.seed_endpoint,
|
||||||
|
)
|
||||||
+ seed_tts_bench_argv(locale=ns.seed_tts_locale)
|
+ seed_tts_bench_argv(locale=ns.seed_tts_locale)
|
||||||
+ [
|
+ [
|
||||||
"--seed-tts-wer-eval",
|
"--seed-tts-wer-eval",
|
||||||
@@ -225,6 +244,8 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
|||||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
if ns.seed_tts_turns_per_session > 1:
|
||||||
|
argv.extend(["--seed-tts-turns-per-session", str(ns.seed_tts_turns_per_session)])
|
||||||
if ns.seed_tts_wer_save_items:
|
if ns.seed_tts_wer_save_items:
|
||||||
argv.append("--seed-tts-wer-save-items")
|
argv.append("--seed-tts-wer-save-items")
|
||||||
if ns.seed_tts_file_ref_audio:
|
if ns.seed_tts_file_ref_audio:
|
||||||
@@ -266,6 +287,23 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
help="If set, forwarded to ``vllm bench serve`` (probe first request until success). "
|
help="If set, forwarded to ``vllm bench serve`` (probe first request until success). "
|
||||||
"Omit to use upstream default (typically skip).",
|
"Omit to use upstream default (typically skip).",
|
||||||
)
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--trust-remote-code",
|
||||||
|
action="store_true",
|
||||||
|
help="Forward trust_remote_code to ``vllm bench serve`` for models with custom code.",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--temperature",
|
||||||
|
type=float,
|
||||||
|
default=None,
|
||||||
|
help="Optional sampling temperature forwarded to ``vllm bench serve`` (e.g. 0 for MiniCPM-o Daily-Omni MCQ).",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--output-len",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help="Optional max output tokens forwarded to ``vllm bench serve`` (e.g. 512 for MiniCPM-o Daily-Omni).",
|
||||||
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--result-dir",
|
"--result-dir",
|
||||||
type=Path,
|
type=Path,
|
||||||
@@ -295,6 +333,13 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
help="Optional local Videos root; if omitted, media is fetched lazily from Hub Videos.tar.",
|
help="Optional local Videos root; if omitted, media is fetched lazily from Hub Videos.tar.",
|
||||||
)
|
)
|
||||||
p.add_argument("--daily-omni-input-mode", choices=("all", "visual", "audio"), default="all")
|
p.add_argument("--daily-omni-input-mode", choices=("all", "visual", "audio"), default="all")
|
||||||
|
p.add_argument(
|
||||||
|
"--daily-omni-pack-mode",
|
||||||
|
choices=("qwen", "minicpm-interleave"),
|
||||||
|
default=None,
|
||||||
|
help="Daily-Omni multimodal pack recipe. Use ``minicpm-interleave`` for MiniCPM-o "
|
||||||
|
"(1fps interleaved image/audio; see PR #5606).",
|
||||||
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--daily-extra-body-json",
|
"--daily-extra-body-json",
|
||||||
default='{"modalities":["text"]}',
|
default='{"modalities":["text"]}',
|
||||||
@@ -326,6 +371,22 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
default='{"modalities":["text","audio"]}',
|
default='{"modalities":["text","audio"]}',
|
||||||
help="JSON for Seed-TTS chat requests (must include audio for synthesis + PCM capture).",
|
help="JSON for Seed-TTS chat requests (must include audio for synthesis + PCM capture).",
|
||||||
)
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--seed-backend",
|
||||||
|
default="openai-chat-omni",
|
||||||
|
help="Seed-TTS benchmark backend (for example openai-realtime-tts).",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--seed-endpoint",
|
||||||
|
default="/v1/chat/completions",
|
||||||
|
help="Seed-TTS serving endpoint corresponding to --seed-backend.",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--seed-tts-turns-per-session",
|
||||||
|
type=int,
|
||||||
|
default=1,
|
||||||
|
help="Group this many Seed-TTS target texts into each Realtime session.",
|
||||||
|
)
|
||||||
p.add_argument("--seed-tts-wer-save-items", action="store_true")
|
p.add_argument("--seed-tts-wer-save-items", action="store_true")
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--seed-tts-file-ref-audio",
|
"--seed-tts-file-ref-audio",
|
||||||
@@ -400,6 +461,11 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
|||||||
min_mean_sim=ns.min_seed_tts_mean_sim,
|
min_mean_sim=ns.min_seed_tts_mean_sim,
|
||||||
min_mean_utmos=ns.min_seed_tts_mean_utmos,
|
min_mean_utmos=ns.min_seed_tts_mean_utmos,
|
||||||
)
|
)
|
||||||
|
if ns.seed_tts_turns_per_session > 1:
|
||||||
|
expected_turns = ns.num_prompts * ns.seed_tts_turns_per_session
|
||||||
|
evaluated_turns = int(data.get("seed_tts_content_evaluated") or 0)
|
||||||
|
if evaluated_turns != expected_turns:
|
||||||
|
errs.append(f"seed_tts_content_evaluated={evaluated_turns} != expected {expected_turns}")
|
||||||
if errs:
|
if errs:
|
||||||
failed.extend([f"[Seed-TTS] {e}" for e in errs])
|
failed.extend([f"[Seed-TTS] {e}" for e in errs])
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ from vllm_omni.experimental.fullduplex.client import (
|
|||||||
read_pcm16_wav,
|
read_pcm16_wav,
|
||||||
write_pcm16_wav,
|
write_pcm16_wav,
|
||||||
)
|
)
|
||||||
|
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
|
||||||
|
MiniCPMO45DuplexPolicy,
|
||||||
|
)
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
@@ -32,6 +35,19 @@ def test_realtime_client_builds_explicit_native_duplex_url():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_seed_tts_initial_text_is_part_of_native_duplex_context():
|
||||||
|
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
|
||||||
|
"Speak exactly.",
|
||||||
|
True,
|
||||||
|
"The quick brown fox.",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert prefix == "<|im_start|>system\nSpeak exactly.\n<|audio_start|>"
|
||||||
|
assert suffix == (
|
||||||
|
"<|audio_end|><|im_end|>\n<|im_start|>user\nThe quick brown fox.<|im_end|>\n<|im_start|>assistant\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_realtime_client_builds_resume_only_url_when_autostart_disabled():
|
def test_realtime_client_builds_resume_only_url_when_autostart_disabled():
|
||||||
url = build_realtime_url(
|
url = build_realtime_url(
|
||||||
"ws://localhost:8099/v1/realtime?duplex=1",
|
"ws://localhost:8099/v1/realtime?duplex=1",
|
||||||
@@ -86,6 +102,59 @@ async def test_realtime_client_configure_sends_explicit_ref_audio():
|
|||||||
assert session["ref_audio"] == "data:audio/wav;base64,AAAA"
|
assert session["ref_audio"] == "data:audio/wav;base64,AAAA"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_realtime_client_configure_sends_seed_tts_text_condition():
|
||||||
|
class Client(RealtimeDuplexClient):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__("ws://unused")
|
||||||
|
self.sent = []
|
||||||
|
|
||||||
|
async def send(self, event):
|
||||||
|
self.sent.append(event)
|
||||||
|
self.events.add({"type": "session.created"})
|
||||||
|
|
||||||
|
client = Client()
|
||||||
|
|
||||||
|
await client.configure(
|
||||||
|
"openbmb/MiniCPM-o-4_5",
|
||||||
|
instructions="Speak the requested text exactly.",
|
||||||
|
initial_user_text="The quick brown fox.",
|
||||||
|
timeout_s=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
session = client.sent[0]["session"]
|
||||||
|
assert session["instructions"] == "Speak the requested text exactly."
|
||||||
|
assert session["extra_body"]["duplex_initial_user_text"] == "The quick brown fox."
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_realtime_client_configure_explicit_tts_opts_out_of_native_duplex():
|
||||||
|
class Client(RealtimeDuplexClient):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__("ws://unused")
|
||||||
|
self.sent = []
|
||||||
|
|
||||||
|
async def send(self, event):
|
||||||
|
self.sent.append(event)
|
||||||
|
self.events.add({"type": "session.created"})
|
||||||
|
|
||||||
|
client = Client()
|
||||||
|
|
||||||
|
await client.configure(
|
||||||
|
"openbmb/MiniCPM-o-4_5",
|
||||||
|
native_duplex=False,
|
||||||
|
auto_response=False,
|
||||||
|
extra_body={"ref_audio": "data:audio/wav;base64,AAAA"},
|
||||||
|
timeout_s=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
session_extra_body = client.sent[0]["session"]["extra_body"]
|
||||||
|
assert session_extra_body == {
|
||||||
|
"ref_audio": "data:audio/wav;base64,AAAA",
|
||||||
|
"minicpmo45_native_duplex": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_realtime_event_collector_partitions_audio_by_response():
|
def test_realtime_event_collector_partitions_audio_by_response():
|
||||||
collector = RealtimeEventCollector()
|
collector = RealtimeEventCollector()
|
||||||
collector.add({"type": "response.created", "response": {"id": "resp-a"}})
|
collector.add({"type": "response.created", "response": {"id": "resp-a"}})
|
||||||
@@ -134,6 +203,26 @@ def test_realtime_event_collector_reports_engine_token_and_audio_intervals():
|
|||||||
},
|
},
|
||||||
received_at_s=received_at_s,
|
received_at_s=received_at_s,
|
||||||
)
|
)
|
||||||
|
collector.add(
|
||||||
|
{
|
||||||
|
"type": "response.audio_transcript.delta",
|
||||||
|
"response_id": "resp-a",
|
||||||
|
"delta": "",
|
||||||
|
},
|
||||||
|
received_at_s=10.1,
|
||||||
|
)
|
||||||
|
collector.add(
|
||||||
|
{
|
||||||
|
"type": "response.audio_transcript.delta",
|
||||||
|
"response_id": "resp-a",
|
||||||
|
"delta": "hello",
|
||||||
|
},
|
||||||
|
received_at_s=10.15,
|
||||||
|
)
|
||||||
|
collector.add(
|
||||||
|
{"type": "response.done", "response": {"id": "resp-a"}},
|
||||||
|
received_at_s=10.4,
|
||||||
|
)
|
||||||
|
|
||||||
timing = collector.timing_summary(
|
timing = collector.timing_summary(
|
||||||
after_s=10.0,
|
after_s=10.0,
|
||||||
@@ -175,6 +264,19 @@ def test_realtime_event_collector_reports_engine_token_and_audio_intervals():
|
|||||||
},
|
},
|
||||||
"max_chunk_gap_ms": 110.0,
|
"max_chunk_gap_ms": 110.0,
|
||||||
}
|
}
|
||||||
|
assert timing["request_metrics"] == {
|
||||||
|
"source": "client_monotonic_receive",
|
||||||
|
"measurement_origin": {
|
||||||
|
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||||
|
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||||
|
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||||
|
},
|
||||||
|
"ttft_ms": 250.0,
|
||||||
|
"ttfp_ms": 300.0,
|
||||||
|
"rtf": pytest.approx(1.916667),
|
||||||
|
"audio_generation_ms": 460.0,
|
||||||
|
"audio_duration_ms": 240.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_response_timing_ignores_unowned_session_level_metrics():
|
def test_response_timing_ignores_unowned_session_level_metrics():
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ def get_question(prompt_type: str = "text") -> str:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||||
def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
||||||
"""Test processing text, generating text output."""
|
"""Test processing text, generating text output."""
|
||||||
@@ -44,7 +44,7 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||||
def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
|
def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
|
||||||
"""Test processing audio, generating text output."""
|
"""Test processing audio, generating text output."""
|
||||||
@@ -57,7 +57,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||||
def test_image_to_text(omni_runner, omni_runner_handler) -> None:
|
def test_image_to_text(omni_runner, omni_runner_handler) -> None:
|
||||||
"""Test processing image, generating text output."""
|
"""Test processing image, generating text output."""
|
||||||
@@ -68,7 +68,7 @@ def test_image_to_text(omni_runner, omni_runner_handler) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||||
def test_video_to_text(omni_runner, omni_runner_handler) -> None:
|
def test_video_to_text(omni_runner, omni_runner_handler) -> None:
|
||||||
"""Test processing video, generating text output."""
|
"""Test processing video, generating text output."""
|
||||||
@@ -79,7 +79,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||||
def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
|
def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||||
"""Test processing text and generating audio through Talker and Code2Wav."""
|
"""Test processing text and generating audio through Talker and Code2Wav."""
|
||||||
@@ -90,7 +90,7 @@ def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||||
def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
|
def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||||
"""Test processing mixed modalities (image + audio), generating audio output."""
|
"""Test processing mixed modalities (image + audio), generating audio output."""
|
||||||
@@ -109,7 +109,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||||
def test_video_to_audio(omni_runner, omni_runner_handler) -> None:
|
def test_video_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||||
"""Test processing video, generating audio output."""
|
"""Test processing video, generating audio output."""
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Shared helpers for online-serving MiniCPM-o duplex E2E tests."""
|
||||||
+12
-7
@@ -19,14 +19,19 @@ DEPLOY_CONFIG = modify_stage_config(
|
|||||||
get_deploy_config_path("minicpmo_4_5_duplex.yaml"),
|
get_deploy_config_path("minicpmo_4_5_duplex.yaml"),
|
||||||
updates={
|
updates={
|
||||||
"base_config": get_deploy_config_path("minicpmo_4_5.yaml"),
|
"base_config": get_deploy_config_path("minicpmo_4_5.yaml"),
|
||||||
|
# Cap per-stage KV so Thinker/Talker/Code2Wav share one GPU with
|
||||||
|
# max_sessions=2. Talker must stay within its 4096 context so the
|
||||||
|
# 0.5 GiB budget passes vLLM's min-KV check at init.
|
||||||
"stages": {
|
"stages": {
|
||||||
0: {"kv_cache_memory_bytes": 6 * 1024 * 1024 * 1024},
|
0: {"kv_cache_memory_bytes": 6 * 1024 * 1024 * 1024},
|
||||||
1: {"kv_cache_memory_bytes": 512 * 1024 * 1024},
|
1: {
|
||||||
|
"max_model_len": 4096,
|
||||||
|
"kv_cache_memory_bytes": 512 * 1024 * 1024,
|
||||||
|
},
|
||||||
2: {"kv_cache_memory_bytes": 256 * 1024 * 1024},
|
2: {"kv_cache_memory_bytes": 256 * 1024 * 1024},
|
||||||
},
|
},
|
||||||
# Platform overrides are applied after ordinary stage settings. Keep
|
# Platform overrides apply after ordinary stage settings and would
|
||||||
# the constrained duplex test's Talker budget ahead of the base CUDA
|
# otherwise reinstate the base CUDA 2 GiB Talker default.
|
||||||
# profile's single-GPU default.
|
|
||||||
"platforms": {
|
"platforms": {
|
||||||
"cuda": {
|
"cuda": {
|
||||||
"stages": [
|
"stages": [
|
||||||
@@ -48,7 +53,7 @@ CORE_DEPLOY_CONFIG = modify_stage_config(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
ASSET_DIR = Path(__file__).resolve().parents[1] / "assets" / "minicpmo_4_5"
|
ASSET_DIR = Path(__file__).resolve().parents[3] / "assets" / "minicpmo_4_5"
|
||||||
RESPONSE_REQUIRED_WAV = ASSET_DIR / "response_required_16k.wav"
|
RESPONSE_REQUIRED_WAV = ASSET_DIR / "response_required_16k.wav"
|
||||||
RESPONSE_REQUIRED_SHA256 = "2e5fd4eb3ee434ce107ee3a0591fa624a33f7683c7462f45fe651c443c9af941"
|
RESPONSE_REQUIRED_SHA256 = "2e5fd4eb3ee434ce107ee3a0591fa624a33f7683c7462f45fe651c443c9af941"
|
||||||
SOFT_INTERRUPT_WAV = ASSET_DIR / "soft_interrupt_16k.wav"
|
SOFT_INTERRUPT_WAV = ASSET_DIR / "soft_interrupt_16k.wav"
|
||||||
@@ -60,7 +65,7 @@ SERVER_PARAMS = [
|
|||||||
OmniServerParams(
|
OmniServerParams(
|
||||||
model=MODEL,
|
model=MODEL,
|
||||||
stage_config_path=DEPLOY_CONFIG,
|
stage_config_path=DEPLOY_CONFIG,
|
||||||
use_stage_cli=True,
|
use_stage_cli=False,
|
||||||
server_args=["--trust-remote-code"],
|
server_args=["--trust-remote-code"],
|
||||||
),
|
),
|
||||||
id="three-stage-single-gpu",
|
id="three-stage-single-gpu",
|
||||||
@@ -71,7 +76,7 @@ CORE_SERVER_PARAMS = [
|
|||||||
OmniServerParams(
|
OmniServerParams(
|
||||||
model=MODEL,
|
model=MODEL,
|
||||||
stage_config_path=CORE_DEPLOY_CONFIG,
|
stage_config_path=CORE_DEPLOY_CONFIG,
|
||||||
use_stage_cli=True,
|
use_stage_cli=False,
|
||||||
server_args=["--trust-remote-code"],
|
server_args=["--trust-remote-code"],
|
||||||
),
|
),
|
||||||
id="three-stage-single-gpu",
|
id="three-stage-single-gpu",
|
||||||
+37
-2
@@ -14,7 +14,7 @@ not a separate model-level barge-in contract.
|
|||||||
|
|
||||||
Run only after a MiniCPM-o 4.5 vLLM-Omni server is up:
|
Run only after a MiniCPM-o 4.5 vLLM-Omni server is up:
|
||||||
|
|
||||||
python tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py \
|
python tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py \
|
||||||
--url ws://localhost:8099/v1/realtime?duplex=1 \
|
--url ws://localhost:8099/v1/realtime?duplex=1 \
|
||||||
--model openbmb/MiniCPM-o-4_5 \
|
--model openbmb/MiniCPM-o-4_5 \
|
||||||
--input-wav input_16k_mono_pcm16.wav \
|
--input-wav input_16k_mono_pcm16.wav \
|
||||||
@@ -37,7 +37,7 @@ import wave
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
_REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||||
if str(_REPO_ROOT) not in sys.path:
|
if str(_REPO_ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(_REPO_ROOT))
|
sys.path.insert(0, str(_REPO_ROOT))
|
||||||
|
|
||||||
@@ -53,6 +53,7 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
|||||||
RealtimeEventCollector,
|
RealtimeEventCollector,
|
||||||
build_realtime_url,
|
build_realtime_url,
|
||||||
read_pcm16_wav,
|
read_pcm16_wav,
|
||||||
|
summarize_session_request_metrics,
|
||||||
)
|
)
|
||||||
|
|
||||||
_url_with_model = build_realtime_url
|
_url_with_model = build_realtime_url
|
||||||
@@ -194,6 +195,38 @@ class DemoState:
|
|||||||
summaries[response_id] = timing
|
summaries[response_id] = timing
|
||||||
return summaries
|
return summaries
|
||||||
|
|
||||||
|
def session_request_metrics(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
) -> list[dict[str, object]]:
|
||||||
|
"""Return benchmark-style metrics for every response in this session."""
|
||||||
|
requests: list[dict[str, object]] = []
|
||||||
|
for request_index, (response_id, timing) in enumerate(self.response_timing_summaries().items()):
|
||||||
|
metrics = timing.get("request_metrics")
|
||||||
|
if not isinstance(metrics, dict):
|
||||||
|
continue
|
||||||
|
requests.append(
|
||||||
|
{
|
||||||
|
"session_id": session_id,
|
||||||
|
"request_index": request_index,
|
||||||
|
"response_id": response_id,
|
||||||
|
**metrics,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return requests
|
||||||
|
|
||||||
|
def session_metric_summary(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Average TTFT, TTFP, and RTF across responses that emitted audio."""
|
||||||
|
return summarize_session_request_metrics(
|
||||||
|
self.session_request_metrics(session_id=session_id),
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
|
||||||
def first_index(self, event_type: str, predicate=None) -> int | None:
|
def first_index(self, event_type: str, predicate=None) -> int | None:
|
||||||
for index, event in enumerate(self.events):
|
for index, event in enumerate(self.events):
|
||||||
if event.get("type") != event_type:
|
if event.get("type") != event_type:
|
||||||
@@ -1538,6 +1571,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
|||||||
"input_transcription_ok": input_transcription_ok,
|
"input_transcription_ok": input_transcription_ok,
|
||||||
"completed_response_ids": completed_response_ids,
|
"completed_response_ids": completed_response_ids,
|
||||||
"response_timings": state.response_timing_summaries(),
|
"response_timings": state.response_timing_summaries(),
|
||||||
|
"request_metrics": state.session_request_metrics(session_id=args.session_id),
|
||||||
|
"session_metrics": state.session_metric_summary(session_id=args.session_id),
|
||||||
"lifecycle_counts_ok": lifecycle_counts_ok,
|
"lifecycle_counts_ok": lifecycle_counts_ok,
|
||||||
"validation_mode": validation_mode,
|
"validation_mode": validation_mode,
|
||||||
"scenario": scenario,
|
"scenario": scenario,
|
||||||
@@ -8,6 +8,7 @@ import hashlib
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import uuid
|
import uuid
|
||||||
|
from functools import lru_cache
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
@@ -16,14 +17,29 @@ import websockets
|
|||||||
from websockets.exceptions import ConnectionClosed
|
from websockets.exceptions import ConnectionClosed
|
||||||
|
|
||||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
if str(SCRIPT_DIR) not in sys.path:
|
REPO_ROOT = SCRIPT_DIR.parents[3]
|
||||||
sys.path.insert(0, str(SCRIPT_DIR))
|
if str(REPO_ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(REPO_ROOT))
|
||||||
|
|
||||||
from minicpmo_realtime_duplex_scenarios import ( # noqa: E402
|
|
||||||
_ref_audio_data_url,
|
@lru_cache(maxsize=1)
|
||||||
_url_with_model,
|
def _scenario_module():
|
||||||
run_demo,
|
"""Delay heavyweight vLLM imports so ``--help`` remains directly executable."""
|
||||||
)
|
from tests.e2e.online_serving.helpers import minicpmo_realtime_duplex_scenarios
|
||||||
|
|
||||||
|
return minicpmo_realtime_duplex_scenarios
|
||||||
|
|
||||||
|
|
||||||
|
def _ref_audio_data_url(path: str) -> str:
|
||||||
|
return _scenario_module()._ref_audio_data_url(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _url_with_model(*args, **kwargs) -> str:
|
||||||
|
return _scenario_module()._url_with_model(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_demo(args):
|
||||||
|
return await _scenario_module().run_demo(args)
|
||||||
|
|
||||||
|
|
||||||
def _with_resume_mode(url: str) -> str:
|
def _with_resume_mode(url: str) -> str:
|
||||||
|
|||||||
@@ -314,6 +314,14 @@ def summarize_artifacts(
|
|||||||
and event["response"].get("status") == "cancelled"
|
and event["response"].get("status") == "cancelled"
|
||||||
)
|
)
|
||||||
result_ok = result.get("ok") is True
|
result_ok = result.get("ok") is True
|
||||||
|
# response-required keeps the full listen sandwich around commit. model-policy
|
||||||
|
# only requires a completed audio response that started before final commit:
|
||||||
|
# single-GPU co-location often still drains speak after the WAV ends.
|
||||||
|
commit_listen_contract_ok = (
|
||||||
|
listen_after_response_before_commit and final_listen_after_commit
|
||||||
|
if validation_mode == "response-required"
|
||||||
|
else response_before_final_commit
|
||||||
|
)
|
||||||
common_contract_ok = bool(
|
common_contract_ok = bool(
|
||||||
result_ok
|
result_ok
|
||||||
and not error_events
|
and not error_events
|
||||||
@@ -323,8 +331,7 @@ def summarize_artifacts(
|
|||||||
and multi_delta_ok
|
and multi_delta_ok
|
||||||
and response_audio_contract_ok
|
and response_audio_contract_ok
|
||||||
and response_before_final_commit
|
and response_before_final_commit
|
||||||
and listen_after_response_before_commit
|
and commit_listen_contract_ok
|
||||||
and final_listen_after_commit
|
|
||||||
)
|
)
|
||||||
mode_contract_ok = validation_mode == "model-policy" or (
|
mode_contract_ok = validation_mode == "model-policy" or (
|
||||||
second_response_before_final_commit
|
second_response_before_final_commit
|
||||||
@@ -393,6 +400,11 @@ async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]:
|
|||||||
f"duplex-soft-interrupt-{uuid.uuid4().hex}",
|
f"duplex-soft-interrupt-{uuid.uuid4().hex}",
|
||||||
]
|
]
|
||||||
command.extend(["--ref-audio", str(_canonical_path(args.ref_audio))])
|
command.extend(["--ref-audio", str(_canonical_path(args.ref_audio))])
|
||||||
|
temperature = getattr(args, "temperature", None)
|
||||||
|
if temperature is None and args.validation_mode == "response-required":
|
||||||
|
temperature = 0.0
|
||||||
|
if temperature is not None:
|
||||||
|
command.extend(["--temperature", str(temperature)])
|
||||||
if args.require_audio:
|
if args.require_audio:
|
||||||
command.append("--require-audio")
|
command.append("--require-audio")
|
||||||
if args.no_realtime_pacing:
|
if args.no_realtime_pacing:
|
||||||
@@ -453,6 +465,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
parser.add_argument("--summary-output")
|
parser.add_argument("--summary-output")
|
||||||
parser.add_argument("--chunk-ms", type=int, default=200)
|
parser.add_argument("--chunk-ms", type=int, default=200)
|
||||||
parser.add_argument("--timeout-s", type=float, default=120.0)
|
parser.add_argument("--timeout-s", type=float, default=120.0)
|
||||||
|
parser.add_argument(
|
||||||
|
"--temperature",
|
||||||
|
type=float,
|
||||||
|
default=None,
|
||||||
|
help="Stage0 sampling temperature; response-required defaults to 0.0.",
|
||||||
|
)
|
||||||
parser.add_argument("--require-audio", action="store_true")
|
parser.add_argument("--require-audio", action="store_true")
|
||||||
parser.add_argument("--no-realtime-pacing", action="store_true")
|
parser.add_argument("--no-realtime-pacing", action="store_true")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
|
|||||||
@@ -68,12 +68,12 @@ def get_max_batch_size(size_type="few"):
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_text_to_text_001(omni_server, openai_client) -> None:
|
def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Test text-only input generating text output via OpenAI API.
|
Test text-only input generating text output via OpenAI API.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text
|
Input Modal: text
|
||||||
Output Modal: text
|
Output Modal: text
|
||||||
Input Setting: stream=False
|
Input Setting: stream=False
|
||||||
@@ -93,13 +93,13 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Test text-only input generating text + audio output via OpenAI API.
|
Test text-only input generating text + audio output via OpenAI API.
|
||||||
This exercises the talker TTS region detection and in-process token2wav vocoder.
|
This exercises the talker TTS region detection and in-process token2wav vocoder.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text
|
Input Modal: text
|
||||||
Output Modal: text + audio
|
Output Modal: text + audio
|
||||||
Input Setting: stream=True
|
Input Setting: stream=True
|
||||||
@@ -118,12 +118,12 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Test audio input generating text + audio output via OpenAI API.
|
Test audio input generating text + audio output via OpenAI API.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text + audio
|
Input Modal: text + audio
|
||||||
Output Modal: text + audio
|
Output Modal: text + audio
|
||||||
Input Setting: stream=True
|
Input Setting: stream=True
|
||||||
@@ -146,12 +146,12 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Test image input generating text + audio output via OpenAI API.
|
Test image input generating text + audio output via OpenAI API.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text + image
|
Input Modal: text + image
|
||||||
Output Modal: text + audio
|
Output Modal: text + audio
|
||||||
Input Setting: stream=True
|
Input Setting: stream=True
|
||||||
@@ -174,12 +174,12 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
|||||||
|
|
||||||
@pytest.mark.full_model
|
@pytest.mark.full_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Test video input generating text + audio output via OpenAI API.
|
Test video input generating text + audio output via OpenAI API.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text + video
|
Input Modal: text + video
|
||||||
Output Modal: text + audio
|
Output Modal: text + audio
|
||||||
Input Setting: stream=True
|
Input Setting: stream=True
|
||||||
@@ -203,12 +203,12 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
|||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Test multi-modal input (text + audio + video + image) generating text + audio output.
|
Test multi-modal input (text + audio + video + image) generating text + audio output.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text + audio + video + image
|
Input Modal: text + audio + video + image
|
||||||
Output Modal: text + audio
|
Output Modal: text + audio
|
||||||
Input Setting: stream=True
|
Input Setting: stream=True
|
||||||
|
|||||||
@@ -10,15 +10,7 @@ from pathlib import Path
|
|||||||
import pytest
|
import pytest
|
||||||
import websockets
|
import websockets
|
||||||
|
|
||||||
from tests.e2e.online_serving.minicpmo_realtime_duplex_scenarios import (
|
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||||
_ref_audio_data_url,
|
|
||||||
run_demo,
|
|
||||||
)
|
|
||||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import (
|
|
||||||
run_multi_session,
|
|
||||||
)
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.minicpmo_4_5_duplex import (
|
|
||||||
CORE_SERVER_PARAMS,
|
CORE_SERVER_PARAMS,
|
||||||
SERVER_PARAMS,
|
SERVER_PARAMS,
|
||||||
demo_args,
|
demo_args,
|
||||||
@@ -27,11 +19,42 @@ from tests.helpers.minicpmo_4_5_duplex import (
|
|||||||
resolve_ref_audio,
|
resolve_ref_audio,
|
||||||
validated_input_wav,
|
validated_input_wav,
|
||||||
)
|
)
|
||||||
|
from tests.e2e.online_serving.helpers.minicpmo_realtime_duplex_scenarios import (
|
||||||
|
_ref_audio_data_url,
|
||||||
|
run_demo,
|
||||||
|
)
|
||||||
|
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import (
|
||||||
|
run_multi_session,
|
||||||
|
)
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
from vllm_omni.experimental.fullduplex.client import build_realtime_url
|
from vllm_omni.experimental.fullduplex.client import build_realtime_url
|
||||||
|
|
||||||
pytestmark = pytest.mark.omni
|
pytestmark = pytest.mark.omni
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_request_metrics(metrics: object, *, expected_count: int) -> None:
|
||||||
|
assert isinstance(metrics, list)
|
||||||
|
assert len(metrics) == expected_count
|
||||||
|
for request_index, request in enumerate(metrics):
|
||||||
|
assert isinstance(request["session_id"], str)
|
||||||
|
assert request["request_index"] == request_index
|
||||||
|
assert isinstance(request["response_id"], str)
|
||||||
|
assert request["ttft_ms"] is not None and request["ttft_ms"] >= 0
|
||||||
|
assert request["ttfp_ms"] >= 0
|
||||||
|
assert request["rtf"] is not None and request["rtf"] >= 0
|
||||||
|
assert request["audio_generation_ms"] >= 0
|
||||||
|
assert request["audio_duration_ms"] > 0
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_session_metrics(metrics: object, *, expected_count: int) -> None:
|
||||||
|
assert isinstance(metrics, dict)
|
||||||
|
assert isinstance(metrics["session_id"], str)
|
||||||
|
assert metrics["audio_turn_count"] == expected_count
|
||||||
|
assert metrics["mean_ttft_ms"] is not None and metrics["mean_ttft_ms"] >= 0
|
||||||
|
assert metrics["mean_ttfp_ms"] is not None and metrics["mean_ttfp_ms"] >= 0
|
||||||
|
assert metrics["mean_rtf"] is not None and metrics["mean_rtf"] >= 0
|
||||||
|
|
||||||
|
|
||||||
async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]:
|
async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]:
|
||||||
async def receive() -> list[dict[str, object]]:
|
async def receive() -> list[dict[str, object]]:
|
||||||
events: list[dict[str, object]] = []
|
events: list[dict[str, object]] = []
|
||||||
@@ -83,7 +106,7 @@ async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", CORE_SERVER_PARAMS, indirect=True)
|
@pytest.mark.parametrize("omni_server", CORE_SERVER_PARAMS, indirect=True)
|
||||||
def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None:
|
def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None:
|
||||||
ref_audio = resolve_ref_audio(model_prefix)
|
ref_audio = resolve_ref_audio(model_prefix)
|
||||||
@@ -101,29 +124,29 @@ def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||||
def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||||
result = asyncio.run(
|
args = demo_args(
|
||||||
run_demo(
|
omni_server=omni_server,
|
||||||
demo_args(
|
input_wav=validated_input_wav(),
|
||||||
omni_server=omni_server,
|
ref_audio=resolve_ref_audio(model_prefix),
|
||||||
input_wav=validated_input_wav(),
|
output_dir=tmp_path / "single_session",
|
||||||
ref_audio=resolve_ref_audio(model_prefix),
|
|
||||||
output_dir=tmp_path / "single_session",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
args.turns = 2
|
||||||
|
result = asyncio.run(run_demo(args))
|
||||||
assert result["ok"] is True, json.dumps(result, ensure_ascii=False, indent=2)
|
assert result["ok"] is True, json.dumps(result, ensure_ascii=False, indent=2)
|
||||||
assert result["audio_delta_count"] > 0
|
assert result["audio_delta_count"] > 0
|
||||||
assert result["done_count"] == 1
|
assert result["done_count"] == 2
|
||||||
assert result["error_count"] == 0
|
assert result["error_count"] == 0
|
||||||
assert result["all_audio_responses_have_transcript"] is True
|
assert result["all_audio_responses_have_transcript"] is True
|
||||||
assert result["transcript_delta_done_ok"] is True
|
assert result["transcript_delta_done_ok"] is True
|
||||||
|
_assert_request_metrics(result["request_metrics"], expected_count=2)
|
||||||
|
_assert_session_metrics(result["session_metrics"], expected_count=2)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||||
def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||||
result = asyncio.run(
|
result = asyncio.run(
|
||||||
@@ -145,3 +168,6 @@ def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str,
|
|||||||
assert all(session["audio_delta_count"] > 0 for session in result["sessions"])
|
assert all(session["audio_delta_count"] > 0 for session in result["sessions"])
|
||||||
assert all(session["done_count"] == 1 for session in result["sessions"])
|
assert all(session["done_count"] == 1 for session in result["sessions"])
|
||||||
assert all(session["error_count"] == 0 for session in result["sessions"])
|
assert all(session["error_count"] == 0 for session in result["sessions"])
|
||||||
|
for session in result["sessions"]:
|
||||||
|
_assert_request_metrics(session["request_metrics"], expected_count=1)
|
||||||
|
_assert_session_metrics(session["session_metrics"], expected_count=1)
|
||||||
@@ -9,14 +9,7 @@ from types import SimpleNamespace
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import (
|
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||||
run_lifecycle_probes,
|
|
||||||
)
|
|
||||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_soft_interrupt import (
|
|
||||||
run_soft_interrupt,
|
|
||||||
)
|
|
||||||
from tests.helpers.mark import hardware_test
|
|
||||||
from tests.helpers.minicpmo_4_5_duplex import (
|
|
||||||
SERVER_PARAMS,
|
SERVER_PARAMS,
|
||||||
SOFT_INTERRUPT_SHA256,
|
SOFT_INTERRUPT_SHA256,
|
||||||
multi_session_args,
|
multi_session_args,
|
||||||
@@ -25,11 +18,18 @@ from tests.helpers.minicpmo_4_5_duplex import (
|
|||||||
validated_input_wav,
|
validated_input_wav,
|
||||||
validated_soft_interrupt_wav,
|
validated_soft_interrupt_wav,
|
||||||
)
|
)
|
||||||
|
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import (
|
||||||
|
run_lifecycle_probes,
|
||||||
|
)
|
||||||
|
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_soft_interrupt import (
|
||||||
|
run_soft_interrupt,
|
||||||
|
)
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
|
|
||||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||||
def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||||
args = multi_session_args(
|
args = multi_session_args(
|
||||||
@@ -53,7 +53,7 @@ def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_
|
|||||||
assert result["admission"]["overflow_error_code"] == "resource_exhausted"
|
assert result["admission"]["overflow_error_code"] == "resource_exhausted"
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||||
def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||||
input_wav = validated_soft_interrupt_wav()
|
input_wav = validated_soft_interrupt_wav()
|
||||||
@@ -70,8 +70,13 @@ def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -
|
|||||||
timeout_s=180.0,
|
timeout_s=180.0,
|
||||||
require_audio=True,
|
require_audio=True,
|
||||||
no_realtime_pacing=False,
|
no_realtime_pacing=False,
|
||||||
validation_mode="response-required",
|
# Single-GPU co-location cannot reliably finish the fixture's
|
||||||
min_responses=2,
|
# two-response soft-interrupt handoff before the WAV ends.
|
||||||
|
# model-policy + deterministic sampling checks streaming audio
|
||||||
|
# lifecycle instead of the diagnostic two-response contract.
|
||||||
|
validation_mode="model-policy",
|
||||||
|
temperature=0.0,
|
||||||
|
min_responses=1,
|
||||||
min_audio_deltas_per_response=2,
|
min_audio_deltas_per_response=2,
|
||||||
input_sha256=SOFT_INTERRUPT_SHA256,
|
input_sha256=SOFT_INTERRUPT_SHA256,
|
||||||
expect_followup_response_substring=None,
|
expect_followup_response_substring=None,
|
||||||
@@ -83,4 +88,5 @@ def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -
|
|||||||
assert result["error_count"] == 0
|
assert result["error_count"] == 0
|
||||||
assert result["response_lifecycle_ok"] is True
|
assert result["response_lifecycle_ok"] is True
|
||||||
assert result["response_audio_contract_ok"] is True
|
assert result["response_audio_contract_ok"] is True
|
||||||
assert result["followup_response_transcript_ok"] is True
|
assert result["response_before_final_commit"] is True
|
||||||
|
assert result["enough_responses"] is True
|
||||||
@@ -63,12 +63,12 @@ def get_max_batch_size(size_type="few"):
|
|||||||
return batch_sizes.get(size_type, 5)
|
return batch_sizes.get(size_type, 5)
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Test text + video input generating text output.
|
Test text + video input generating text output.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text + video
|
Input Modal: text + video
|
||||||
Output Modal: text
|
Output Modal: text
|
||||||
Input Setting: stream=False
|
Input Setting: stream=False
|
||||||
@@ -90,13 +90,13 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
|||||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Verify that sequential requests produce independent results and that the
|
Verify that sequential requests produce independent results and that the
|
||||||
second request does not receive the first request's audio or token state.
|
second request does not receive the first request's audio or token state.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text (two different prompts)
|
Input Modal: text (two different prompts)
|
||||||
Output Modal: text + audio (both)
|
Output Modal: text + audio (both)
|
||||||
"""
|
"""
|
||||||
@@ -131,13 +131,13 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Test text input generating a longer audio output to exercise the
|
Test text input generating a longer audio output to exercise the
|
||||||
Code2Wav stage across multiple frames.
|
Code2Wav stage across multiple frames.
|
||||||
Deploy Setting: default 2GPU
|
Deploy Setting: default single GPU
|
||||||
Input Modal: text (longer prompt)
|
Input Modal: text (longer prompt)
|
||||||
Output Modal: text + audio
|
Output Modal: text + audio
|
||||||
Input Setting: stream=True
|
Input Setting: stream=True
|
||||||
@@ -166,7 +166,7 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
|||||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_chinese_text_to_audio(omni_server, openai_client) -> None:
|
def test_chinese_text_to_audio(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ import pytest
|
|||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
DRIVER_DIR = Path(__file__).resolve().parent
|
DRIVER_DIR = Path(__file__).resolve().parent
|
||||||
DEMO_PATH = DRIVER_DIR / "minicpmo_realtime_duplex_scenarios.py"
|
HELPERS_DIR = DRIVER_DIR / "helpers"
|
||||||
|
DEMO_PATH = HELPERS_DIR / "minicpmo_realtime_duplex_scenarios.py"
|
||||||
MULTI_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_multi_session.py"
|
MULTI_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_multi_session.py"
|
||||||
PAIR_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_demo_pair.py"
|
PAIR_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_demo_pair.py"
|
||||||
SOFT_INTERRUPT_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_soft_interrupt.py"
|
SOFT_INTERRUPT_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_soft_interrupt.py"
|
||||||
@@ -333,6 +334,71 @@ def test_realtime_duplex_soft_interrupt_accepts_explicit_ref_audio(monkeypatch):
|
|||||||
|
|
||||||
assert args.ref_audio == "ref.wav"
|
assert args.ref_audio == "ref.wav"
|
||||||
assert args.validation_mode == "model-policy"
|
assert args.validation_mode == "model-policy"
|
||||||
|
assert args.temperature is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_realtime_duplex_soft_interrupt_response_required_defaults_temperature(tmp_path, monkeypatch):
|
||||||
|
demo = _load_soft_interrupt_demo_module()
|
||||||
|
input_wav = tmp_path / "input.wav"
|
||||||
|
with wave.open(str(input_wav), "wb") as wav_file:
|
||||||
|
wav_file.setnchannels(1)
|
||||||
|
wav_file.setsampwidth(2)
|
||||||
|
wav_file.setframerate(16_000)
|
||||||
|
wav_file.writeframes(b"\x00\x00" * 320)
|
||||||
|
ref_audio = tmp_path / "ref.wav"
|
||||||
|
ref_audio.write_bytes(b"ref")
|
||||||
|
output_dir = tmp_path / "out"
|
||||||
|
output_dir.mkdir()
|
||||||
|
captured: dict[str, object] = {}
|
||||||
|
|
||||||
|
class _FakeProcess:
|
||||||
|
returncode = 0
|
||||||
|
|
||||||
|
async def communicate(self):
|
||||||
|
return b'{"ok": true}', b""
|
||||||
|
|
||||||
|
def kill(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _fake_exec(*command, **kwargs):
|
||||||
|
captured["command"] = list(command)
|
||||||
|
return _FakeProcess()
|
||||||
|
|
||||||
|
monkeypatch.setattr(demo.asyncio, "create_subprocess_exec", _fake_exec)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
demo,
|
||||||
|
"summarize_artifacts",
|
||||||
|
lambda **kwargs: {"ok": True, "returncode": 0},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
demo.run_soft_interrupt(
|
||||||
|
SimpleNamespace(
|
||||||
|
url="ws://127.0.0.1:8099/v1/realtime?duplex=1",
|
||||||
|
model="openbmb/MiniCPM-o-4_5",
|
||||||
|
input_wav=str(input_wav),
|
||||||
|
ref_audio=str(ref_audio),
|
||||||
|
output_dir=str(output_dir),
|
||||||
|
summary_output=None,
|
||||||
|
chunk_ms=200,
|
||||||
|
timeout_s=5.0,
|
||||||
|
require_audio=True,
|
||||||
|
no_realtime_pacing=False,
|
||||||
|
validation_mode="response-required",
|
||||||
|
temperature=None,
|
||||||
|
min_responses=2,
|
||||||
|
min_audio_deltas_per_response=2,
|
||||||
|
input_sha256=None,
|
||||||
|
expect_followup_response_substring=None,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
command = captured["command"]
|
||||||
|
assert isinstance(command, list)
|
||||||
|
assert "--temperature" in command
|
||||||
|
assert command[command.index("--temperature") + 1] == "0.0"
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
|
||||||
def test_realtime_duplex_soft_interrupt_response_required_needs_bound_fixture(monkeypatch):
|
def test_realtime_duplex_soft_interrupt_response_required_needs_bound_fixture(monkeypatch):
|
||||||
@@ -497,6 +563,43 @@ def test_realtime_duplex_soft_interrupt_model_policy_accepts_single_response(tmp
|
|||||||
assert summary["listen_after_response_before_commit"] is True
|
assert summary["listen_after_response_before_commit"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_realtime_duplex_soft_interrupt_model_policy_accepts_commit_during_speak(tmp_path):
|
||||||
|
demo = _load_soft_interrupt_demo_module()
|
||||||
|
output = tmp_path / "model_policy_commit_during_speak"
|
||||||
|
output.mkdir()
|
||||||
|
response_id = "resp-long"
|
||||||
|
events = [
|
||||||
|
{"type": "response.listen", "_client_received_at_s": 1.0},
|
||||||
|
{"type": "response.created", "response": {"id": response_id}, "_client_received_at_s": 2.0},
|
||||||
|
{"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.1},
|
||||||
|
{"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.2},
|
||||||
|
{"type": "input_audio_buffer.committed", "_client_received_at_s": 3.0},
|
||||||
|
{"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 4.0},
|
||||||
|
{"type": "response.done", "response_id": response_id, "_client_received_at_s": 5.0},
|
||||||
|
]
|
||||||
|
(output / "events.jsonl").write_text(
|
||||||
|
"".join(demo.json.dumps(event) + "\n" for event in events),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(output / "result.json").write_text(
|
||||||
|
demo.json.dumps({"ok": True, "response_ids": [response_id]}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
summary = demo.summarize_artifacts(
|
||||||
|
output_dir=output,
|
||||||
|
validation_mode="model-policy",
|
||||||
|
min_responses=1,
|
||||||
|
min_audio_deltas_per_response=2,
|
||||||
|
expect_followup_response_substring=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert summary["ok"] is True
|
||||||
|
assert summary["response_before_final_commit"] is True
|
||||||
|
assert summary["listen_after_response_before_commit"] is False
|
||||||
|
assert summary["final_listen_after_commit"] is False
|
||||||
|
|
||||||
|
|
||||||
def test_realtime_duplex_soft_interrupt_response_required_rejects_single_response(tmp_path):
|
def test_realtime_duplex_soft_interrupt_response_required_rejects_single_response(tmp_path):
|
||||||
demo = _load_soft_interrupt_demo_module()
|
demo = _load_soft_interrupt_demo_module()
|
||||||
output = tmp_path / "response_required"
|
output = tmp_path / "response_required"
|
||||||
@@ -799,6 +902,7 @@ def test_realtime_duplex_demo_reads_response_playback_cursor():
|
|||||||
def test_realtime_duplex_demo_partitions_timing_by_response_identity():
|
def test_realtime_duplex_demo_partitions_timing_by_response_identity():
|
||||||
demo = _load_demo_module()
|
demo = _load_demo_module()
|
||||||
state = demo.DemoState()
|
state = demo.DemoState()
|
||||||
|
state.input_commit_sent_at_s.extend((9.9, 19.9))
|
||||||
for response_id, created_at_s, audio_at_s, token_count in (
|
for response_id, created_at_s, audio_at_s, token_count in (
|
||||||
("resp-1", 10.0, 10.1, 3),
|
("resp-1", 10.0, 10.1, 3),
|
||||||
("resp-2", 20.0, 20.2, 5),
|
("resp-2", 20.0, 20.2, 5),
|
||||||
@@ -826,15 +930,66 @@ def test_realtime_duplex_demo_partitions_timing_by_response_identity():
|
|||||||
},
|
},
|
||||||
received_at_s=audio_at_s,
|
received_at_s=audio_at_s,
|
||||||
)
|
)
|
||||||
|
state.add(
|
||||||
|
{
|
||||||
|
"type": "response.audio_transcript.delta",
|
||||||
|
"response_id": response_id,
|
||||||
|
"delta": "audio",
|
||||||
|
},
|
||||||
|
received_at_s=created_at_s + 0.05,
|
||||||
|
)
|
||||||
|
|
||||||
timings = state.response_timing_summaries()
|
timings = state.response_timing_summaries()
|
||||||
|
requests = state.session_request_metrics(session_id="seed-tts-session")
|
||||||
|
session_metrics = state.session_metric_summary(session_id="seed-tts-session")
|
||||||
|
|
||||||
assert timings["resp-1"]["stage0_tokens"]["output_token_count"] == 3
|
assert timings["resp-1"]["stage0_tokens"]["output_token_count"] == 3
|
||||||
assert timings["resp-1"]["audio_output"]["response_created_to_first_audio_ms"] == 100.0
|
assert timings["resp-1"]["audio_output"]["response_created_to_first_audio_ms"] == 100.0
|
||||||
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] is None
|
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] == 200.0
|
||||||
assert timings["resp-2"]["stage0_tokens"]["output_token_count"] == 5
|
assert timings["resp-2"]["stage0_tokens"]["output_token_count"] == 5
|
||||||
assert timings["resp-2"]["audio_output"]["response_created_to_first_audio_ms"] == 200.0
|
assert timings["resp-2"]["audio_output"]["response_created_to_first_audio_ms"] == 200.0
|
||||||
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] is None
|
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] == 300.0
|
||||||
|
assert requests == [
|
||||||
|
{
|
||||||
|
"session_id": "seed-tts-session",
|
||||||
|
"request_index": 0,
|
||||||
|
"response_id": "resp-1",
|
||||||
|
"ttft_ms": 150.0,
|
||||||
|
"ttfp_ms": 200.0,
|
||||||
|
"rtf": 2.5,
|
||||||
|
"audio_generation_ms": 200.0,
|
||||||
|
"audio_duration_ms": 80.0,
|
||||||
|
"source": "client_monotonic_receive",
|
||||||
|
"measurement_origin": {
|
||||||
|
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||||
|
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||||
|
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"session_id": "seed-tts-session",
|
||||||
|
"request_index": 1,
|
||||||
|
"response_id": "resp-2",
|
||||||
|
"ttft_ms": 150.0,
|
||||||
|
"ttfp_ms": 300.0,
|
||||||
|
"rtf": 3.75,
|
||||||
|
"audio_generation_ms": 300.0,
|
||||||
|
"audio_duration_ms": 80.0,
|
||||||
|
"source": "client_monotonic_receive",
|
||||||
|
"measurement_origin": {
|
||||||
|
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||||
|
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||||
|
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
assert session_metrics == {
|
||||||
|
"session_id": "seed-tts-session",
|
||||||
|
"audio_turn_count": 2,
|
||||||
|
"mean_ttft_ms": 150.0,
|
||||||
|
"mean_ttfp_ms": 250.0,
|
||||||
|
"mean_rtf": 3.125,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_realtime_duplex_demo_model_policy_accepts_one_listen_per_streamed_turn():
|
def test_realtime_duplex_demo_model_policy_accepts_one_listen_per_streamed_turn():
|
||||||
|
|||||||
@@ -340,6 +340,56 @@ def test_strip_single_engine_args_model_does_not_trigger_warning(mocker):
|
|||||||
assert "model" not in warned_args
|
assert "model" not in warned_args
|
||||||
|
|
||||||
|
|
||||||
|
def test_new_format_deploy_yaml_does_not_strip_interleave_cli_overrides(tmp_path):
|
||||||
|
"""New-format deploy YAMLs must not strip interleave_mm_strings / media_io_kwargs.
|
||||||
|
|
||||||
|
Pytest historically passed ``--stage-configs-path`` for deploy YAMLs; stripping
|
||||||
|
those CLI flags dropped MiniCPM interleaved AV packing and cut Daily-Omni
|
||||||
|
accuracy from ~78% to ~70%.
|
||||||
|
"""
|
||||||
|
from vllm_omni.entrypoints.utils import is_new_format_deploy_config
|
||||||
|
|
||||||
|
yaml_path = tmp_path / "minicpm_deploy.yaml"
|
||||||
|
yaml_path.write_text(
|
||||||
|
"""
|
||||||
|
pipeline: minicpmo_4_5
|
||||||
|
async_chunk: true
|
||||||
|
stages:
|
||||||
|
- stage_id: 0
|
||||||
|
devices: "0"
|
||||||
|
""".strip(),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
path = str(yaml_path)
|
||||||
|
assert is_new_format_deploy_config(path)
|
||||||
|
|
||||||
|
kwargs = {
|
||||||
|
"stage_configs_path": path,
|
||||||
|
"trust_remote_code": True,
|
||||||
|
"interleave_mm_strings": True,
|
||||||
|
"media_io_kwargs": {"video": {"fps": 1, "num_frames": 128}},
|
||||||
|
"gpu_memory_utilization": 0.55,
|
||||||
|
}
|
||||||
|
# Mirror AsyncOmniEngine._resolve_stage_configs strip decision.
|
||||||
|
if kwargs.get("stage_configs_path") is not None and not is_new_format_deploy_config(kwargs["stage_configs_path"]):
|
||||||
|
base = AsyncOmniEngine._strip_single_engine_args(dict(kwargs))
|
||||||
|
else:
|
||||||
|
base = dict(kwargs)
|
||||||
|
assert base.get("interleave_mm_strings") is True
|
||||||
|
assert base.get("media_io_kwargs") == {"video": {"fps": 1, "num_frames": 128}}
|
||||||
|
# Parent EngineArgs fields also remain available for deploy merge.
|
||||||
|
assert base.get("gpu_memory_utilization") == 0.55
|
||||||
|
|
||||||
|
legacy = tmp_path / "legacy.yaml"
|
||||||
|
legacy.write_text("stage_args:\n - stage_id: 0\n", encoding="utf-8")
|
||||||
|
assert not is_new_format_deploy_config(str(legacy))
|
||||||
|
legacy_kwargs = dict(kwargs, stage_configs_path=str(legacy))
|
||||||
|
stripped = AsyncOmniEngine._strip_single_engine_args(legacy_kwargs)
|
||||||
|
assert "interleave_mm_strings" not in stripped
|
||||||
|
assert "media_io_kwargs" not in stripped
|
||||||
|
assert "gpu_memory_utilization" not in stripped
|
||||||
|
|
||||||
|
|
||||||
# For https://github.com/vllm-project/vllm-omni/issues/3293
|
# For https://github.com/vllm-project/vllm-omni/issues/3293
|
||||||
def test_tensor_parallel_size_none_is_handled():
|
def test_tensor_parallel_size_none_is_handled():
|
||||||
"""Ensure the tensor parallel size of None isn't forwarded."""
|
"""Ensure the tensor parallel size of None isn't forwarded."""
|
||||||
|
|||||||
@@ -51,6 +51,19 @@ from vllm_omni.outputs import OmniRequestOutput
|
|||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
def test_native_input_append_supports_explicit_session_opt_out():
|
||||||
|
session = DuplexSession(
|
||||||
|
"sid-explicit-tts",
|
||||||
|
DuplexSessionConfig(model="test-model"),
|
||||||
|
capabilities=DuplexCapabilities.minicpmo45_native(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert OmniDuplexSessionHandler._uses_native_input_append(session) is True
|
||||||
|
|
||||||
|
session.config.extra_body["minicpmo45_native_duplex"] = False
|
||||||
|
assert OmniDuplexSessionHandler._uses_native_input_append(session) is False
|
||||||
|
|
||||||
|
|
||||||
class _ModelConfig:
|
class _ModelConfig:
|
||||||
model = "test-model"
|
model = "test-model"
|
||||||
|
|
||||||
|
|||||||
@@ -3179,7 +3179,12 @@ def iter_omni_server(
|
|||||||
print("OmniServer stopping...")
|
print("OmniServer stopping...")
|
||||||
else:
|
else:
|
||||||
if stage_config_path is not None:
|
if stage_config_path is not None:
|
||||||
server_args += ["--deploy-config", stage_config_path]
|
from vllm_omni.entrypoints.utils import is_new_format_deploy_config
|
||||||
|
|
||||||
|
if is_new_format_deploy_config(stage_config_path):
|
||||||
|
server_args += ["--deploy-config", stage_config_path]
|
||||||
|
else:
|
||||||
|
server_args += ["--stage-configs-path", stage_config_path]
|
||||||
|
|
||||||
with (
|
with (
|
||||||
OmniServer(
|
OmniServer(
|
||||||
|
|||||||
@@ -168,7 +168,7 @@ class TestDeployTopology:
|
|||||||
assert memory_utilizations == [
|
assert memory_utilizations == [
|
||||||
0.55,
|
0.55,
|
||||||
0.15,
|
0.15,
|
||||||
0.15,
|
0.18,
|
||||||
]
|
]
|
||||||
assert sum(memory_utilizations) <= 0.9 + 1e-6
|
assert sum(memory_utilizations) <= 0.9 + 1e-6
|
||||||
# Daily-Omni minicpm-interleave: up to 64 image/audio items (+ optional video).
|
# Daily-Omni minicpm-interleave: up to 64 image/audio items (+ optional video).
|
||||||
|
|||||||
@@ -55,6 +55,16 @@ class SeedTTSSampleRequest(SampleRequest):
|
|||||||
seed_tts_system_prompt: str = ""
|
seed_tts_system_prompt: str = ""
|
||||||
#: Local path to reference prompt WAV (for SIM vs. synthesized PCM in ``seed_tts_eval``).
|
#: Local path to reference prompt WAV (for SIM vs. synthesized PCM in ``seed_tts_eval``).
|
||||||
seed_tts_ref_wav_path: str = ""
|
seed_tts_ref_wav_path: str = ""
|
||||||
|
#: Ordered text targets evaluated as turns in one Realtime session.
|
||||||
|
seed_tts_turns: tuple[SeedTTSTurn, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SeedTTSTurn:
|
||||||
|
"""One target utterance within a grouped Realtime Seed-TTS session."""
|
||||||
|
|
||||||
|
utterance_id: str
|
||||||
|
target_text: str
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -216,24 +226,39 @@ class SeedTTSDataset(BenchmarkDataset):
|
|||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
if output_len is None:
|
if output_len is None:
|
||||||
output_len = self.DEFAULT_OUTPUT_LEN
|
output_len = self.DEFAULT_OUTPUT_LEN
|
||||||
|
turns_per_session = int(kwargs.pop("turns_per_session", 1))
|
||||||
|
if turns_per_session < 1:
|
||||||
|
raise ValueError("turns_per_session must be at least 1")
|
||||||
|
|
||||||
tok = get_cached_tokenizer(tokenizer)
|
tok = get_cached_tokenizer(tokenizer)
|
||||||
out: list[SampleRequest] = []
|
out: list[SampleRequest] = []
|
||||||
for i, row in enumerate(self._rows):
|
valid_rows: list[tuple[_SeedTTSRow, Path]] = []
|
||||||
if len(out) >= num_requests:
|
for row in self._rows:
|
||||||
break
|
|
||||||
wav_path = (self._root / self.locale / row.prompt_wav_rel).resolve()
|
wav_path = (self._root / self.locale / row.prompt_wav_rel).resolve()
|
||||||
if not wav_path.is_file():
|
if not wav_path.is_file():
|
||||||
logger.warning("Missing prompt wav for %s: %s", row.utterance_id, wav_path)
|
logger.warning("Missing prompt wav for %s: %s", row.utterance_id, wav_path)
|
||||||
continue
|
continue
|
||||||
|
valid_rows.append((row, wav_path))
|
||||||
|
|
||||||
target = row.target_text
|
for session_index in range(num_requests):
|
||||||
prompt_len = len(tok.encode(f"{self._system_prompt}\n{target}"))
|
offset = session_index * turns_per_session
|
||||||
|
group = valid_rows[offset : offset + turns_per_session]
|
||||||
|
if len(group) < turns_per_session:
|
||||||
|
break
|
||||||
|
reference_row, reference_wav_path = group[0]
|
||||||
|
turns = tuple(
|
||||||
|
SeedTTSTurn(
|
||||||
|
utterance_id=row.utterance_id,
|
||||||
|
target_text=row.target_text,
|
||||||
|
)
|
||||||
|
for row, _ in group
|
||||||
|
)
|
||||||
|
prompt_len = sum(len(tok.encode(f"{self._system_prompt}\n{turn.target_text}")) for turn in turns)
|
||||||
lang = "English" if self.locale == "en" else "Chinese"
|
lang = "English" if self.locale == "en" else "Chinese"
|
||||||
ref_uri = _ref_audio_payload(wav_path, inline=self.inline_ref_audio)
|
ref_uri = _ref_audio_payload(reference_wav_path, inline=self.inline_ref_audio)
|
||||||
speech_extra: dict[str, Any] = {
|
speech_extra: dict[str, Any] = {
|
||||||
"ref_audio": ref_uri,
|
"ref_audio": ref_uri,
|
||||||
"ref_text": row.ref_text,
|
"ref_text": reference_row.ref_text,
|
||||||
"task_type": "Base",
|
"task_type": "Base",
|
||||||
"language": lang,
|
"language": lang,
|
||||||
"max_new_tokens": output_len,
|
"max_new_tokens": output_len,
|
||||||
@@ -241,20 +266,26 @@ class SeedTTSDataset(BenchmarkDataset):
|
|||||||
|
|
||||||
out.append(
|
out.append(
|
||||||
SeedTTSSampleRequest(
|
SeedTTSSampleRequest(
|
||||||
prompt=target,
|
prompt=turns[0].target_text,
|
||||||
prompt_len=prompt_len,
|
prompt_len=prompt_len,
|
||||||
expected_output_len=output_len,
|
expected_output_len=output_len,
|
||||||
multi_modal_data=None,
|
multi_modal_data=None,
|
||||||
request_id=f"{request_id_prefix}{i}",
|
request_id=f"{request_id_prefix}{session_index}",
|
||||||
seed_tts_speech_extra=speech_extra,
|
seed_tts_speech_extra=speech_extra,
|
||||||
seed_tts_utterance_id=row.utterance_id,
|
seed_tts_utterance_id=turns[0].utterance_id,
|
||||||
seed_tts_locale=self.locale,
|
seed_tts_locale=self.locale,
|
||||||
seed_tts_system_prompt=self._system_prompt,
|
seed_tts_system_prompt=self._system_prompt,
|
||||||
seed_tts_ref_wav_path=str(wav_path),
|
seed_tts_ref_wav_path=str(reference_wav_path),
|
||||||
|
seed_tts_turns=turns,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Seed-TTS: built %d requests (asked %d)", len(out), num_requests)
|
logger.info(
|
||||||
|
"Seed-TTS: built %d sessions (asked %d, turns_per_session=%d)",
|
||||||
|
len(out),
|
||||||
|
num_requests,
|
||||||
|
turns_per_session,
|
||||||
|
)
|
||||||
self.maybe_oversample_requests(out, num_requests, request_id_prefix, no_oversample)
|
self.maybe_oversample_requests(out, num_requests, request_id_prefix, no_oversample)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|||||||
@@ -54,6 +54,8 @@ import string
|
|||||||
import tempfile
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
import wave
|
import wave
|
||||||
|
from copy import copy
|
||||||
|
from dataclasses import replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -123,11 +125,25 @@ def _get_eval_device() -> str:
|
|||||||
return explicit
|
return explicit
|
||||||
try:
|
try:
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
return "cuda:0" if torch.cuda.is_available() else "cpu"
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
return "cpu"
|
return "cpu"
|
||||||
|
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
return "cuda:0"
|
||||||
|
|
||||||
|
# Ascend: ``torch.npu`` only exists once ``torch_npu`` has been imported, which does
|
||||||
|
# not necessarily happen in the benchmark client process. Without this branch the
|
||||||
|
# eval silently lands on CPU, where Whisper-large-v3 needs hours for a full Seed-TTS
|
||||||
|
# run and the CI job is killed by its timeout instead of reporting a WER.
|
||||||
|
try:
|
||||||
|
import torch_npu # noqa: F401
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
if hasattr(torch, "npu") and torch.npu.is_available():
|
||||||
|
return "npu:0"
|
||||||
|
|
||||||
|
return "cpu"
|
||||||
|
|
||||||
|
|
||||||
def _punctuation_all() -> str:
|
def _punctuation_all() -> str:
|
||||||
from zhon.hanzi import punctuation
|
from zhon.hanzi import punctuation
|
||||||
@@ -512,6 +528,42 @@ def _missing_deps_message(lang: str) -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _expand_seed_tts_turn_outputs(
|
||||||
|
input_requests: list[SeedTTSSampleRequest],
|
||||||
|
outputs: list[Any],
|
||||||
|
) -> tuple[list[SeedTTSSampleRequest], list[Any]]:
|
||||||
|
"""Expand grouped Realtime sessions into one request/output pair per turn."""
|
||||||
|
expanded_requests: list[SeedTTSSampleRequest] = []
|
||||||
|
expanded_outputs: list[Any] = []
|
||||||
|
for request, output in zip(input_requests, outputs, strict=True):
|
||||||
|
turns = request.seed_tts_turns
|
||||||
|
if not turns:
|
||||||
|
expanded_requests.append(request)
|
||||||
|
expanded_outputs.append(output)
|
||||||
|
continue
|
||||||
|
turn_pcm = getattr(output, "tts_turn_pcm_bytes", None)
|
||||||
|
session_pcm = getattr(output, "tts_output_pcm_bytes", None)
|
||||||
|
for turn_index, turn in enumerate(turns):
|
||||||
|
expanded_requests.append(
|
||||||
|
replace(
|
||||||
|
request,
|
||||||
|
prompt=turn.target_text,
|
||||||
|
seed_tts_utterance_id=turn.utterance_id,
|
||||||
|
seed_tts_turns=(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
turn_output = copy(output)
|
||||||
|
if isinstance(turn_pcm, list) and turn_index < len(turn_pcm):
|
||||||
|
turn_output.tts_output_pcm_bytes = turn_pcm[turn_index]
|
||||||
|
elif len(turns) == 1:
|
||||||
|
# openai-chat-omni Seed-TTS only fills the session-level PCM field.
|
||||||
|
turn_output.tts_output_pcm_bytes = session_pcm
|
||||||
|
else:
|
||||||
|
turn_output.tts_output_pcm_bytes = None
|
||||||
|
expanded_outputs.append(turn_output)
|
||||||
|
return expanded_requests, expanded_outputs
|
||||||
|
|
||||||
|
|
||||||
def compute_seed_tts_wer_metrics(
|
def compute_seed_tts_wer_metrics(
|
||||||
input_requests: list[SampleRequest],
|
input_requests: list[SampleRequest],
|
||||||
outputs: list[Any],
|
outputs: list[Any],
|
||||||
@@ -524,6 +576,8 @@ def compute_seed_tts_wer_metrics(
|
|||||||
return None
|
return None
|
||||||
if not all(isinstance(r, SeedTTSSampleRequest) for r in input_requests):
|
if not all(isinstance(r, SeedTTSSampleRequest) for r in input_requests):
|
||||||
return None
|
return None
|
||||||
|
session_count = len(input_requests)
|
||||||
|
input_requests, outputs = _expand_seed_tts_turn_outputs(input_requests, outputs)
|
||||||
|
|
||||||
first = input_requests[0]
|
first = input_requests[0]
|
||||||
assert isinstance(first, SeedTTSSampleRequest)
|
assert isinstance(first, SeedTTSSampleRequest)
|
||||||
@@ -759,6 +813,8 @@ def compute_seed_tts_wer_metrics(
|
|||||||
|
|
||||||
result: dict[str, Any] = {
|
result: dict[str, Any] = {
|
||||||
"seed_tts_eval_protocol": "seed-tts-eval",
|
"seed_tts_eval_protocol": "seed-tts-eval",
|
||||||
|
"seed_tts_session_count": session_count,
|
||||||
|
"seed_tts_turn_count": len(input_requests),
|
||||||
"seed_tts_content_evaluated": len(errs),
|
"seed_tts_content_evaluated": len(errs),
|
||||||
"seed_tts_content_error_mean": statistics.fmean(errs) if errs else None,
|
"seed_tts_content_error_mean": statistics.fmean(errs) if errs else None,
|
||||||
"seed_tts_content_error_median": statistics.median(errs) if errs else None,
|
"seed_tts_content_error_median": statistics.median(errs) if errs else None,
|
||||||
|
|||||||
@@ -9,11 +9,13 @@ import ssl
|
|||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
|
import uuid
|
||||||
import wave
|
import wave
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -47,6 +49,11 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
|||||||
)
|
)
|
||||||
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
||||||
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
||||||
|
from vllm_omni.experimental.fullduplex.client import (
|
||||||
|
RealtimeDuplexClient,
|
||||||
|
summarize_session_request_metrics,
|
||||||
|
wait_for,
|
||||||
|
)
|
||||||
from vllm_omni.metrics import definitions as defs
|
from vllm_omni.metrics import definitions as defs
|
||||||
from vllm_omni.metrics.utils import coerce_positive_int_scalar
|
from vllm_omni.metrics.utils import coerce_positive_int_scalar
|
||||||
|
|
||||||
@@ -188,6 +195,9 @@ def _attach_seed_tts_to_request_func_input(sample: SampleRequest, rfi: RequestFu
|
|||||||
# Mark for PCM capture (WER / UTMOS eval) regardless of extra body presence.
|
# Mark for PCM capture (WER / UTMOS eval) regardless of extra body presence.
|
||||||
setattr(rfi, "seed_tts_row", True)
|
setattr(rfi, "seed_tts_row", True)
|
||||||
sys_prompt = (sample.seed_tts_system_prompt or "").strip() or SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT
|
sys_prompt = (sample.seed_tts_system_prompt or "").strip() or SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT
|
||||||
|
setattr(rfi, "seed_tts_system_prompt", sys_prompt)
|
||||||
|
setattr(rfi, "seed_tts_speech_extra", sample.seed_tts_speech_extra)
|
||||||
|
setattr(rfi, "seed_tts_turns", sample.seed_tts_turns)
|
||||||
setattr(
|
setattr(
|
||||||
rfi,
|
rfi,
|
||||||
"omni_chat_messages",
|
"omni_chat_messages",
|
||||||
@@ -235,7 +245,12 @@ def get_samples(args, tokenizer):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Check if we need to handle omni-related backends/datasets
|
# Check if we need to handle omni-related backends/datasets
|
||||||
is_omni_backend = args.backend in ["openai-chat-omni", "openai-audio-speech", "daily-omni"]
|
is_omni_backend = args.backend in [
|
||||||
|
"openai-chat-omni",
|
||||||
|
"openai-audio-speech",
|
||||||
|
"openai-realtime-duplex",
|
||||||
|
"daily-omni",
|
||||||
|
]
|
||||||
is_omni_dataset = is_daily_omni or is_seed_tts or args.dataset_name == "random-mm"
|
is_omni_dataset = is_daily_omni or is_seed_tts or args.dataset_name == "random-mm"
|
||||||
|
|
||||||
if not is_omni_backend and not is_omni_dataset:
|
if not is_omni_backend and not is_omni_dataset:
|
||||||
@@ -334,10 +349,17 @@ def get_samples(args, tokenizer):
|
|||||||
return input_requests
|
return input_requests
|
||||||
|
|
||||||
if is_seed_tts:
|
if is_seed_tts:
|
||||||
if args.backend not in ("openai-audio-speech", "openai-chat-omni"):
|
if args.backend not in (
|
||||||
|
"openai-audio-speech",
|
||||||
|
"openai-chat-omni",
|
||||||
|
"openai-realtime-duplex",
|
||||||
|
"openai-realtime-tts",
|
||||||
|
):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Seed-TTS requires --backend openai-audio-speech (POST /v1/audio/speech) or "
|
"Seed-TTS requires --backend openai-audio-speech (POST /v1/audio/speech) or "
|
||||||
"--backend openai-chat-omni (POST /v1/chat/completions with ref_audio/ref_text). "
|
"--backend openai-chat-omni (POST /v1/chat/completions with ref_audio/ref_text), or "
|
||||||
|
"--backend openai-realtime-duplex or openai-realtime-tts "
|
||||||
|
"(WebSocket /v1/realtime). "
|
||||||
f"Got backend={args.backend!r}."
|
f"Got backend={args.backend!r}."
|
||||||
)
|
)
|
||||||
repo_id = getattr(args, "dataset_path", None) or getattr(args, "hf_name", None)
|
repo_id = getattr(args, "dataset_path", None) or getattr(args, "hf_name", None)
|
||||||
@@ -346,6 +368,14 @@ def get_samples(args, tokenizer):
|
|||||||
"Seed-TTS requires --dataset-path (HF dataset repo id or local directory) or "
|
"Seed-TTS requires --dataset-path (HF dataset repo id or local directory) or "
|
||||||
"--hf-name for the Hub dataset id."
|
"--hf-name for the Hub dataset id."
|
||||||
)
|
)
|
||||||
|
turns_per_session = int(getattr(args, "seed_tts_turns_per_session", 1))
|
||||||
|
if turns_per_session > 1 and args.backend not in {
|
||||||
|
"openai-realtime-duplex",
|
||||||
|
"openai-realtime-tts",
|
||||||
|
}:
|
||||||
|
raise ValueError(
|
||||||
|
f"--seed-tts-turns-per-session > 1 requires a Realtime Seed-TTS backend. Got backend={args.backend!r}."
|
||||||
|
)
|
||||||
|
|
||||||
_cls_map = {
|
_cls_map = {
|
||||||
"seed-tts": SeedTTSDataset,
|
"seed-tts": SeedTTSDataset,
|
||||||
@@ -375,6 +405,7 @@ def get_samples(args, tokenizer):
|
|||||||
output_len=out_len,
|
output_len=out_len,
|
||||||
request_id_prefix=args.request_id_prefix,
|
request_id_prefix=args.request_id_prefix,
|
||||||
no_oversample=args.no_oversample,
|
no_oversample=args.no_oversample,
|
||||||
|
turns_per_session=turns_per_session,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Handle random-mm dataset (Omni's synthetic multimodal dataset)
|
# Handle random-mm dataset (Omni's synthetic multimodal dataset)
|
||||||
@@ -430,10 +461,14 @@ class MixRequestFuncOutput(RequestFuncOutput):
|
|||||||
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
|
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
|
||||||
#: resampled export after ``openai-chat-omni`` audio deltas.
|
#: resampled export after ``openai-chat-omni`` audio deltas.
|
||||||
tts_output_pcm_bytes: bytes | None = None
|
tts_output_pcm_bytes: bytes | None = None
|
||||||
|
#: Per-turn 24 kHz mono PCM for grouped Realtime Seed-TTS WER.
|
||||||
|
tts_turn_pcm_bytes: list[bytes] | None = None
|
||||||
#: Per-stage snapshot from orchestrator ``metrics["stage_metrics"]`` (merged across SSE chunks).
|
#: Per-stage snapshot from orchestrator ``metrics["stage_metrics"]`` (merged across SSE chunks).
|
||||||
stage_metrics: dict[str, dict] | None = None
|
stage_metrics: dict[str, dict] | None = None
|
||||||
stage_id: int | None = None
|
stage_id: int | None = None
|
||||||
final_output_type: str | None = None
|
final_output_type: str | None = None
|
||||||
|
duplex_request_metrics: list[dict[str, object]] | None = None
|
||||||
|
duplex_session_metrics: dict[str, object] | None = None
|
||||||
|
|
||||||
|
|
||||||
_IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS = (
|
_IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS = (
|
||||||
@@ -1249,6 +1284,180 @@ async def async_request_openai_audio_speech(
|
|||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
def _realtime_websocket_url(api_url: str) -> str:
|
||||||
|
parts = urlsplit(api_url)
|
||||||
|
scheme = "wss" if parts.scheme == "https" else "ws"
|
||||||
|
query = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||||||
|
query.setdefault("duplex", "1")
|
||||||
|
return urlunsplit((scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))
|
||||||
|
|
||||||
|
|
||||||
|
async def async_request_openai_realtime_duplex(
|
||||||
|
request_func_input: RequestFuncInput,
|
||||||
|
session: aiohttp.ClientSession,
|
||||||
|
pbar: tqdm | None = None,
|
||||||
|
) -> MixRequestFuncOutput:
|
||||||
|
"""Run one or more Seed-TTS turns through one explicit Realtime TTS session."""
|
||||||
|
del session
|
||||||
|
output = MixRequestFuncOutput()
|
||||||
|
output.prompt_len = request_func_input.prompt_len
|
||||||
|
output.start_time = time.perf_counter()
|
||||||
|
speech_extra = getattr(request_func_input, "seed_tts_speech_extra", None)
|
||||||
|
if not isinstance(speech_extra, dict):
|
||||||
|
speech_extra = {}
|
||||||
|
configured_turns = getattr(request_func_input, "seed_tts_turns", ())
|
||||||
|
turn_prompts = [
|
||||||
|
(
|
||||||
|
str(getattr(turn, "utterance_id", "") or ""),
|
||||||
|
str(getattr(turn, "target_text", "") or ""),
|
||||||
|
)
|
||||||
|
for turn in configured_turns
|
||||||
|
if str(getattr(turn, "target_text", "") or "").strip()
|
||||||
|
]
|
||||||
|
if not turn_prompts:
|
||||||
|
turn_prompts = [("", request_func_input.prompt)]
|
||||||
|
session_id = f"seed-tts-{request_func_input.request_id or uuid.uuid4().hex}"
|
||||||
|
try:
|
||||||
|
async with RealtimeDuplexClient(_realtime_websocket_url(request_func_input.api_url)) as client:
|
||||||
|
await client.configure(
|
||||||
|
request_func_input.model_name or request_func_input.model,
|
||||||
|
output_audio_format="pcm16",
|
||||||
|
instructions=getattr(
|
||||||
|
request_func_input,
|
||||||
|
"seed_tts_system_prompt",
|
||||||
|
SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT,
|
||||||
|
),
|
||||||
|
native_duplex=False,
|
||||||
|
auto_response=False,
|
||||||
|
extra_body=speech_extra,
|
||||||
|
session_id=session_id,
|
||||||
|
timeout_s=120.0,
|
||||||
|
)
|
||||||
|
turn_metrics: list[dict[str, object]] = []
|
||||||
|
turn_timings: list[dict[str, object]] = []
|
||||||
|
turn_pcm_bytes: list[bytes] = []
|
||||||
|
turn_transcripts: list[str] = []
|
||||||
|
measurement_origin = {
|
||||||
|
"ttft": "conversation.item.create client send to first non-empty text delta",
|
||||||
|
"ttfp": "conversation.item.create client send to first audio packet",
|
||||||
|
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
|
||||||
|
}
|
||||||
|
for request_index, (utterance_id, target_text) in enumerate(turn_prompts):
|
||||||
|
response_offset = len(client.events.response_ids)
|
||||||
|
done_before = client.events.count("response.done")
|
||||||
|
errors_before = len(client.events.errors())
|
||||||
|
turn_started_at_s = time.monotonic()
|
||||||
|
await client.send(
|
||||||
|
{
|
||||||
|
"type": "conversation.item.create",
|
||||||
|
"item": {
|
||||||
|
"type": "message",
|
||||||
|
"role": "user",
|
||||||
|
"content": [{"type": "input_text", "text": target_text}],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
await client.send({"type": "response.create"})
|
||||||
|
await wait_for(
|
||||||
|
lambda: client.events.count("response.done") > done_before
|
||||||
|
or len(client.events.errors()) > errors_before,
|
||||||
|
timeout_s=180.0,
|
||||||
|
label=f"Seed-TTS Realtime TTS turn {request_index} response.done",
|
||||||
|
)
|
||||||
|
errors = client.events.errors()
|
||||||
|
if len(errors) > errors_before:
|
||||||
|
raise RuntimeError(f"Seed-TTS Realtime TTS server error: {errors[-1]}")
|
||||||
|
new_audio_response_ids = [
|
||||||
|
response_id
|
||||||
|
for response_id in client.events.response_ids[response_offset:]
|
||||||
|
if client.events.audio_bytes(response_id)
|
||||||
|
]
|
||||||
|
if len(new_audio_response_ids) != 1:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Seed-TTS Realtime TTS turn {request_index} expected one audio response, "
|
||||||
|
f"got {len(new_audio_response_ids)}"
|
||||||
|
)
|
||||||
|
response_id = new_audio_response_ids[0]
|
||||||
|
timing = client.events.timing_summary(
|
||||||
|
after_s=turn_started_at_s,
|
||||||
|
input_committed_at_s=turn_started_at_s,
|
||||||
|
response_id=response_id,
|
||||||
|
measurement_origin=measurement_origin,
|
||||||
|
)
|
||||||
|
request_metrics = timing.get("request_metrics")
|
||||||
|
if not isinstance(request_metrics, dict):
|
||||||
|
raise RuntimeError(f"Seed-TTS duplex audio turn {response_id} omitted per-request metrics")
|
||||||
|
turn_timings.append(timing)
|
||||||
|
turn_metrics.append(
|
||||||
|
{
|
||||||
|
"session_id": session_id,
|
||||||
|
"request_index": request_index,
|
||||||
|
"utterance_id": utterance_id or None,
|
||||||
|
"response_id": response_id,
|
||||||
|
**request_metrics,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
response_audio = client.events.audio_bytes(response_id)
|
||||||
|
turn_pcm_bytes.append(
|
||||||
|
_pcm_s16le_to_seed_tts_wer_bytes(
|
||||||
|
response_audio,
|
||||||
|
sample_rate=client.events.output_sample_rate_hz,
|
||||||
|
channels=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
turn_transcripts.append(
|
||||||
|
"".join(
|
||||||
|
str(event.get("delta") or "")
|
||||||
|
for event in client.events.events
|
||||||
|
if client.events.response_id(event) == response_id
|
||||||
|
and event.get("type")
|
||||||
|
in {
|
||||||
|
"response.audio_transcript.delta",
|
||||||
|
"response.output_text.delta",
|
||||||
|
"response.text.delta",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await client.acknowledge_playback()
|
||||||
|
request_finished_at = time.perf_counter()
|
||||||
|
session_metrics = summarize_session_request_metrics(
|
||||||
|
turn_metrics,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
await client.close_session(timeout_s=30.0)
|
||||||
|
|
||||||
|
output.generated_text = " ".join(filter(None, turn_transcripts))
|
||||||
|
output.ttft = float(session_metrics.get("mean_ttft_ms") or 0.0) / 1000.0
|
||||||
|
output.audio_ttfp = float(session_metrics.get("mean_ttfp_ms") or 0.0) / 1000.0
|
||||||
|
output.audio_rtf = float(session_metrics.get("mean_rtf") or 0.0)
|
||||||
|
output.audio_duration = (
|
||||||
|
sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0
|
||||||
|
)
|
||||||
|
output.audio_frames = int(output.audio_duration * client.events.output_sample_rate_hz)
|
||||||
|
output.latency = request_finished_at - output.start_time
|
||||||
|
output.tts_turn_pcm_bytes = turn_pcm_bytes
|
||||||
|
output.tts_output_pcm_bytes = b"".join(turn_pcm_bytes)
|
||||||
|
if bool((request_func_input.extra_body or {}).get("save_duplex_request_metrics")):
|
||||||
|
output.duplex_request_metrics = turn_metrics
|
||||||
|
output.duplex_session_metrics = session_metrics
|
||||||
|
output.output_tokens = sum(
|
||||||
|
int(stage0.get("output_token_count") or 0)
|
||||||
|
for timing in turn_timings
|
||||||
|
if isinstance((stage0 := timing.get("stage0_tokens")), dict)
|
||||||
|
)
|
||||||
|
output.success = True
|
||||||
|
except Exception:
|
||||||
|
output.success = False
|
||||||
|
output.error = traceback.format_exc()
|
||||||
|
logger.error(
|
||||||
|
"Seed-TTS Realtime TTS request failed: %s",
|
||||||
|
output.error,
|
||||||
|
)
|
||||||
|
if pbar:
|
||||||
|
pbar.update(1)
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
ASYNC_REQUEST_FUNCS["openai-chat-omni"] = async_request_openai_chat_omni_completions
|
ASYNC_REQUEST_FUNCS["openai-chat-omni"] = async_request_openai_chat_omni_completions
|
||||||
if "openai-chat-omni" not in OPENAI_COMPATIBLE_BACKENDS:
|
if "openai-chat-omni" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-chat-omni")
|
OPENAI_COMPATIBLE_BACKENDS.append("openai-chat-omni")
|
||||||
@@ -1257,6 +1466,13 @@ ASYNC_REQUEST_FUNCS["openai-audio-speech"] = async_request_openai_audio_speech
|
|||||||
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
|
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
|
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
|
||||||
|
|
||||||
|
ASYNC_REQUEST_FUNCS["openai-realtime-duplex"] = async_request_openai_realtime_duplex
|
||||||
|
if "openai-realtime-duplex" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||||
|
OPENAI_COMPATIBLE_BACKENDS.append("openai-realtime-duplex")
|
||||||
|
ASYNC_REQUEST_FUNCS["openai-realtime-tts"] = async_request_openai_realtime_duplex
|
||||||
|
if "openai-realtime-tts" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||||
|
OPENAI_COMPATIBLE_BACKENDS.append("openai-realtime-tts")
|
||||||
|
|
||||||
ASYNC_REQUEST_FUNCS["openai-image-edits-omni"] = async_request_openai_image_edits_omni
|
ASYNC_REQUEST_FUNCS["openai-image-edits-omni"] = async_request_openai_image_edits_omni
|
||||||
if "openai-image-edits-omni" not in OPENAI_COMPATIBLE_BACKENDS:
|
if "openai-image-edits-omni" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-image-edits-omni")
|
OPENAI_COMPATIBLE_BACKENDS.append("openai-image-edits-omni")
|
||||||
@@ -1603,6 +1819,14 @@ async def benchmark(
|
|||||||
"input_lens": [output.prompt_len for output in outputs],
|
"input_lens": [output.prompt_len for output in outputs],
|
||||||
"errors": [output.error for output in outputs],
|
"errors": [output.error for output in outputs],
|
||||||
}
|
}
|
||||||
|
duplex_request_metrics = [metric for output in outputs for metric in (output.duplex_request_metrics or [])]
|
||||||
|
if duplex_request_metrics:
|
||||||
|
result["duplex_request_metrics"] = duplex_request_metrics
|
||||||
|
duplex_session_metrics = [
|
||||||
|
output.duplex_session_metrics for output in outputs if output.duplex_session_metrics is not None
|
||||||
|
]
|
||||||
|
if duplex_session_metrics:
|
||||||
|
result["duplex_session_metrics"] = duplex_session_metrics
|
||||||
|
|
||||||
from vllm_omni.benchmarks.data_modules.daily_omni_eval import (
|
from vllm_omni.benchmarks.data_modules.daily_omni_eval import (
|
||||||
compute_daily_omni_accuracy_metrics,
|
compute_daily_omni_accuracy_metrics,
|
||||||
|
|||||||
@@ -94,6 +94,8 @@ class _ModelEngineOverrides(TypedDict, total=False):
|
|||||||
moe_backend: str
|
moe_backend: str
|
||||||
hf_overrides: Any
|
hf_overrides: Any
|
||||||
limit_mm_per_prompt: dict[str, Any]
|
limit_mm_per_prompt: dict[str, Any]
|
||||||
|
interleave_mm_strings: bool
|
||||||
|
media_io_kwargs: dict[str, Any]
|
||||||
active_stream_window: int
|
active_stream_window: int
|
||||||
enable_sleep_mode: bool
|
enable_sleep_mode: bool
|
||||||
subtalker_sampling_params: dict[str, Any]
|
subtalker_sampling_params: dict[str, Any]
|
||||||
@@ -322,6 +324,9 @@ class OmniStageModelConfig:
|
|||||||
moe_backend: str = "auto"
|
moe_backend: str = "auto"
|
||||||
hf_overrides: Any = None
|
hf_overrides: Any = None
|
||||||
limit_mm_per_prompt: dict[str, Any] | None = None
|
limit_mm_per_prompt: dict[str, Any] | None = None
|
||||||
|
# MiniCPM interleaved AV packing and media decode knobs (Daily-Omni).
|
||||||
|
interleave_mm_strings: bool | None = None
|
||||||
|
media_io_kwargs: dict[str, Any] | None = None
|
||||||
active_stream_window: int = Field(default=0, ge=0)
|
active_stream_window: int = Field(default=0, ge=0)
|
||||||
duplex_max_sessions: int = Field(default=1, ge=1)
|
duplex_max_sessions: int = Field(default=1, ge=1)
|
||||||
enable_sleep_mode: bool = False
|
enable_sleep_mode: bool = False
|
||||||
|
|||||||
@@ -23,11 +23,19 @@ connectors:
|
|||||||
stages:
|
stages:
|
||||||
- stage_id: 0
|
- stage_id: 0
|
||||||
max_num_seqs: 4
|
max_num_seqs: 4
|
||||||
|
# Leave headroom on the shared GPU for Talker KV + concurrent Code2Wav.
|
||||||
gpu_memory_utilization: 0.55
|
gpu_memory_utilization: 0.55
|
||||||
enforce_eager: false
|
enforce_eager: false
|
||||||
tensor_parallel_size: 1
|
tensor_parallel_size: 1
|
||||||
max_num_batched_tokens: 16384
|
max_num_batched_tokens: 16384
|
||||||
devices: "0"
|
devices: "0"
|
||||||
|
# Required for MiniCPM interleaved image/audio Daily-Omni packs (~78% recipe).
|
||||||
|
# Keep these in YAML so they survive both --deploy-config and --stage-configs-path.
|
||||||
|
interleave_mm_strings: true
|
||||||
|
media_io_kwargs:
|
||||||
|
video:
|
||||||
|
fps: 1
|
||||||
|
num_frames: 128
|
||||||
limit_mm_per_prompt:
|
limit_mm_per_prompt:
|
||||||
# Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps).
|
# Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps).
|
||||||
image: 64
|
image: 64
|
||||||
@@ -49,6 +57,8 @@ stages:
|
|||||||
gpu_memory_utilization: 0.15
|
gpu_memory_utilization: 0.15
|
||||||
enforce_eager: false
|
enforce_eager: false
|
||||||
max_num_batched_tokens: 8192
|
max_num_batched_tokens: 8192
|
||||||
|
# Talker context is 4096 (tts_config.max_position_embeddings).
|
||||||
|
max_model_len: 4096
|
||||||
devices: "0"
|
devices: "0"
|
||||||
output_connectors:
|
output_connectors:
|
||||||
to_stage_2: connector_of_shared_memory
|
to_stage_2: connector_of_shared_memory
|
||||||
@@ -66,7 +76,7 @@ stages:
|
|||||||
|
|
||||||
- stage_id: 2
|
- stage_id: 2
|
||||||
max_num_seqs: 4
|
max_num_seqs: 4
|
||||||
gpu_memory_utilization: 0.15
|
gpu_memory_utilization: 0.18
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
enable_chunked_prefill: false
|
enable_chunked_prefill: false
|
||||||
max_num_batched_tokens: 65536
|
max_num_batched_tokens: 65536
|
||||||
|
|||||||
@@ -31,6 +31,11 @@ stages:
|
|||||||
tensor_parallel_size: 1
|
tensor_parallel_size: 1
|
||||||
max_num_batched_tokens: 16384
|
max_num_batched_tokens: 16384
|
||||||
devices: "0"
|
devices: "0"
|
||||||
|
interleave_mm_strings: true
|
||||||
|
media_io_kwargs:
|
||||||
|
video:
|
||||||
|
fps: 1
|
||||||
|
num_frames: 128
|
||||||
limit_mm_per_prompt:
|
limit_mm_per_prompt:
|
||||||
video: 1
|
video: 1
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
|
|||||||
@@ -43,6 +43,11 @@ stages:
|
|||||||
max_model_len: 4096
|
max_model_len: 4096
|
||||||
max_num_batched_tokens: 4096
|
max_num_batched_tokens: 4096
|
||||||
devices: "0,1,2,3"
|
devices: "0,1,2,3"
|
||||||
|
interleave_mm_strings: true
|
||||||
|
media_io_kwargs:
|
||||||
|
video:
|
||||||
|
fps: 1
|
||||||
|
num_frames: 128
|
||||||
limit_mm_per_prompt:
|
limit_mm_per_prompt:
|
||||||
video: 1
|
video: 1
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ stages:
|
|||||||
- stage_id: 1
|
- stage_id: 1
|
||||||
max_num_seqs: 2
|
max_num_seqs: 2
|
||||||
async_scheduling: false
|
async_scheduling: false
|
||||||
|
# Talker context is 4096 (tts_config.max_position_embeddings).
|
||||||
|
max_model_len: 4096
|
||||||
# The split Talker exposes a binary continue/stop vocabulary; native
|
# The split Talker exposes a binary continue/stop vocabulary; native
|
||||||
# segment and turn boundaries travel as explicit output metadata.
|
# segment and turn boundaries travel as explicit output metadata.
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ from vllm_omni.engine.stage_runtime import (
|
|||||||
)
|
)
|
||||||
from vllm_omni.entrypoints.pd_utils import PDDisaggregationMixin
|
from vllm_omni.entrypoints.pd_utils import PDDisaggregationMixin
|
||||||
from vllm_omni.entrypoints.utils import (
|
from vllm_omni.entrypoints.utils import (
|
||||||
|
is_new_format_deploy_config,
|
||||||
load_and_resolve_stage_configs,
|
load_and_resolve_stage_configs,
|
||||||
parse_stage_overrides,
|
parse_stage_overrides,
|
||||||
)
|
)
|
||||||
@@ -1193,7 +1194,12 @@ class AsyncOmniEngine:
|
|||||||
"Ignoring it and resolving stages from stage_configs_path/model factory."
|
"Ignoring it and resolving stages from stage_configs_path/model factory."
|
||||||
)
|
)
|
||||||
|
|
||||||
if stage_configs_path is not None:
|
# Legacy ``stage_args`` YAMLs own per-stage EngineArgs completely, so
|
||||||
|
# strip parent CLI fields. New-format deploy YAMLs (also accepted via
|
||||||
|
# ``--stage-configs-path``) merge CLI overrides the same way as
|
||||||
|
# ``--deploy-config`` — do not strip, or flags like
|
||||||
|
# ``interleave_mm_strings`` / ``media_io_kwargs`` are silently dropped.
|
||||||
|
if stage_configs_path is not None and not is_new_format_deploy_config(stage_configs_path):
|
||||||
base_kwargs = self._strip_single_engine_args(kwargs)
|
base_kwargs = self._strip_single_engine_args(kwargs)
|
||||||
else:
|
else:
|
||||||
base_kwargs = kwargs
|
base_kwargs = kwargs
|
||||||
|
|||||||
@@ -132,6 +132,13 @@ def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None:
|
|||||||
default="en",
|
default="en",
|
||||||
help="Which Seed-TTS split to load: en/meta.lst or zh/meta.lst under the dataset root.",
|
help="Which Seed-TTS split to load: en/meta.lst or zh/meta.lst under the dataset root.",
|
||||||
)
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--seed-tts-turns-per-session",
|
||||||
|
type=int,
|
||||||
|
default=1,
|
||||||
|
help="Group this many Seed-TTS target texts into one Realtime session. "
|
||||||
|
"The first row's reference audio and transcript are reused for every turn.",
|
||||||
|
)
|
||||||
group.add_argument(
|
group.add_argument(
|
||||||
"--seed-tts-root",
|
"--seed-tts-root",
|
||||||
type=str,
|
type=str,
|
||||||
|
|||||||
@@ -30,6 +30,15 @@ PROJECT_ROOT = Path(__file__).parent.parent.parent
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def is_new_format_deploy_config(path: str | None) -> bool:
|
||||||
|
"""Return True when *path* is a deploy YAML (``stages:`` without legacy ``stage_args:``)."""
|
||||||
|
if not path or not os.path.exists(path):
|
||||||
|
return False
|
||||||
|
with open(path, encoding="utf-8") as f:
|
||||||
|
peek = yaml.safe_load(f) or {}
|
||||||
|
return isinstance(peek, dict) and "stages" in peek and "stage_args" not in peek
|
||||||
|
|
||||||
|
|
||||||
_DIFFUSERS_CLASS_TO_CONFIG: dict[str, str] = {
|
_DIFFUSERS_CLASS_TO_CONFIG: dict[str, str] = {
|
||||||
"GlmImagePipeline": "glm_image",
|
"GlmImagePipeline": "glm_image",
|
||||||
}
|
}
|
||||||
@@ -642,9 +651,7 @@ def load_and_resolve_stage_configs(
|
|||||||
"Legacy `stage_configs/` yamls were replaced by `vllm_omni/deploy/<model>.yaml`; "
|
"Legacy `stage_configs/` yamls were replaced by `vllm_omni/deploy/<model>.yaml`; "
|
||||||
"use --deploy-config. See docs/configuration/stage_configs.md."
|
"use --deploy-config. See docs/configuration/stage_configs.md."
|
||||||
)
|
)
|
||||||
with open(stage_configs_path, encoding="utf-8") as f:
|
if is_new_format_deploy_config(stage_configs_path):
|
||||||
_peek = yaml.safe_load(f) or {}
|
|
||||||
if "stages" in _peek and "stage_args" not in _peek:
|
|
||||||
deploy_config_path = stage_configs_path
|
deploy_config_path = stage_configs_path
|
||||||
stage_configs_path = None
|
stage_configs_path = None
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ from pathlib import Path
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
|
|
||||||
|
from vllm_omni.metrics.definitions import compute_audio_rtf
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import websockets
|
import websockets
|
||||||
from websockets.exceptions import ConnectionClosed
|
from websockets.exceptions import ConnectionClosed
|
||||||
@@ -46,6 +48,30 @@ def _interval_summary(values: list[float]) -> dict[str, float | int]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def summarize_session_request_metrics(
|
||||||
|
request_metrics: list[dict[str, object]],
|
||||||
|
*,
|
||||||
|
session_id: str | None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Average client-observed metrics across turns that emitted audio."""
|
||||||
|
|
||||||
|
def mean(metric: str, *, digits: int = 3) -> float | None:
|
||||||
|
values = [
|
||||||
|
float(request[metric])
|
||||||
|
for request in request_metrics
|
||||||
|
if isinstance(request.get(metric), int | float) and math.isfinite(float(request[metric]))
|
||||||
|
]
|
||||||
|
return round(sum(values) / len(values), digits) if values else None
|
||||||
|
|
||||||
|
return {
|
||||||
|
"session_id": session_id,
|
||||||
|
"audio_turn_count": len(request_metrics),
|
||||||
|
"mean_ttft_ms": mean("ttft_ms"),
|
||||||
|
"mean_ttfp_ms": mean("ttfp_ms"),
|
||||||
|
"mean_rtf": mean("rtf", digits=6),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _event_stage_metrics(event: dict[str, object]) -> dict[str, object] | None:
|
def _event_stage_metrics(event: dict[str, object]) -> dict[str, object] | None:
|
||||||
candidates: list[object] = [event.get("vllm_omni")]
|
candidates: list[object] = [event.get("vllm_omni")]
|
||||||
metadata = event.get("metadata")
|
metadata = event.get("metadata")
|
||||||
@@ -205,10 +231,12 @@ class RealtimeEventCollector:
|
|||||||
after_s: float,
|
after_s: float,
|
||||||
input_committed_at_s: float | None = None,
|
input_committed_at_s: float | None = None,
|
||||||
response_id: str | None = None,
|
response_id: str | None = None,
|
||||||
|
measurement_origin: dict[str, str] | None = None,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
"""Summarize engine token metrics and client-observed audio cadence."""
|
"""Summarize engine token metrics and client-observed audio cadence."""
|
||||||
stage0_metrics: dict[str, object] | None = None
|
stage0_metrics: dict[str, object] | None = None
|
||||||
response_created_at_s: float | None = None
|
response_created_at_s: float | None = None
|
||||||
|
first_text_received_at_s: float | None = None
|
||||||
audio_received_at_s: list[float] = []
|
audio_received_at_s: list[float] = []
|
||||||
cumulative_audio_ms: list[float] = []
|
cumulative_audio_ms: list[float] = []
|
||||||
for event, received_at_s in zip(self.events, self.event_received_at_s, strict=True):
|
for event, received_at_s in zip(self.events, self.event_received_at_s, strict=True):
|
||||||
@@ -219,6 +247,18 @@ class RealtimeEventCollector:
|
|||||||
continue
|
continue
|
||||||
if event.get("type") == "response.created" and response_created_at_s is None:
|
if event.get("type") == "response.created" and response_created_at_s is None:
|
||||||
response_created_at_s = received_at_s
|
response_created_at_s = received_at_s
|
||||||
|
if (
|
||||||
|
event.get("type")
|
||||||
|
in {
|
||||||
|
"response.audio_transcript.delta",
|
||||||
|
"response.output_text.delta",
|
||||||
|
"response.text.delta",
|
||||||
|
}
|
||||||
|
and isinstance(event.get("delta"), str)
|
||||||
|
and bool(event["delta"])
|
||||||
|
and first_text_received_at_s is None
|
||||||
|
):
|
||||||
|
first_text_received_at_s = received_at_s
|
||||||
|
|
||||||
stage_metrics = _event_stage_metrics(event)
|
stage_metrics = _event_stage_metrics(event)
|
||||||
stage0 = stage_metrics.get("0") if isinstance(stage_metrics, dict) else None
|
stage0 = stage_metrics.get("0") if isinstance(stage_metrics, dict) else None
|
||||||
@@ -283,6 +323,45 @@ class RealtimeEventCollector:
|
|||||||
"chunk_duration_ms": _interval_summary(chunk_durations_ms),
|
"chunk_duration_ms": _interval_summary(chunk_durations_ms),
|
||||||
"max_chunk_gap_ms": interval_summary["max"],
|
"max_chunk_gap_ms": interval_summary["max"],
|
||||||
}
|
}
|
||||||
|
request_started_at_s = input_committed_at_s if input_committed_at_s is not None else response_created_at_s
|
||||||
|
if request_started_at_s is not None:
|
||||||
|
audio_duration_ms = (
|
||||||
|
max(cumulative_audio_ms)
|
||||||
|
if cumulative_audio_ms
|
||||||
|
else len(self.audio_bytes(response_id))
|
||||||
|
* 1000.0
|
||||||
|
/ (self.output_sample_rate_hz * PCM16_BYTES_PER_SAMPLE)
|
||||||
|
)
|
||||||
|
audio_generation_ms = max(
|
||||||
|
0.0,
|
||||||
|
(audio_received_at_s[-1] - request_started_at_s) * 1000.0,
|
||||||
|
)
|
||||||
|
result["request_metrics"] = {
|
||||||
|
"source": "client_monotonic_receive",
|
||||||
|
"measurement_origin": measurement_origin
|
||||||
|
or {
|
||||||
|
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||||
|
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||||
|
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||||
|
},
|
||||||
|
"ttft_ms": (
|
||||||
|
_rounded_ms((first_text_received_at_s - request_started_at_s) * 1000.0)
|
||||||
|
if first_text_received_at_s is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"ttfp_ms": _rounded_ms((audio_received_at_s[0] - request_started_at_s) * 1000.0),
|
||||||
|
"rtf": round(
|
||||||
|
compute_audio_rtf(
|
||||||
|
audio_generation_ms / 1000.0,
|
||||||
|
audio_duration_ms / 1000.0,
|
||||||
|
),
|
||||||
|
6,
|
||||||
|
)
|
||||||
|
if audio_duration_ms > 0
|
||||||
|
else None,
|
||||||
|
"audio_generation_ms": _rounded_ms(audio_generation_ms),
|
||||||
|
"audio_duration_ms": _rounded_ms(audio_duration_ms),
|
||||||
|
}
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -332,9 +411,26 @@ class RealtimeDuplexClient:
|
|||||||
*,
|
*,
|
||||||
output_audio_format: str = "pcm16",
|
output_audio_format: str = "pcm16",
|
||||||
ref_audio: str | None = None,
|
ref_audio: str | None = None,
|
||||||
|
instructions: str | None = None,
|
||||||
|
initial_user_text: str | None = None,
|
||||||
|
native_duplex: bool = True,
|
||||||
|
auto_response: bool = True,
|
||||||
|
temperature: float | None = None,
|
||||||
|
extra_body: dict[str, object] | None = None,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
timeout_s: float = 20.0,
|
timeout_s: float = 20.0,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
session_extra_body = dict(extra_body or {})
|
||||||
|
if native_duplex:
|
||||||
|
session_extra_body.update(
|
||||||
|
{
|
||||||
|
"auto_response": auto_response,
|
||||||
|
"minicpmo45_native_duplex": True,
|
||||||
|
"force_listen_count": 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
session_extra_body["minicpmo45_native_duplex"] = False
|
||||||
session: dict[str, object] = {
|
session: dict[str, object] = {
|
||||||
"model": model,
|
"model": model,
|
||||||
"modalities": ["audio", "text"],
|
"modalities": ["audio", "text"],
|
||||||
@@ -343,14 +439,18 @@ class RealtimeDuplexClient:
|
|||||||
"turn_detection": None,
|
"turn_detection": None,
|
||||||
"overlap_policy": "listen_only",
|
"overlap_policy": "listen_only",
|
||||||
"playback_commit_policy": "ack_only",
|
"playback_commit_policy": "ack_only",
|
||||||
"extra_body": {
|
"extra_body": session_extra_body,
|
||||||
"auto_response": True,
|
|
||||||
"minicpmo45_native_duplex": True,
|
|
||||||
"force_listen_count": 0,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
if temperature is not None:
|
||||||
|
session["temperature"] = float(temperature)
|
||||||
if ref_audio is not None:
|
if ref_audio is not None:
|
||||||
session["ref_audio"] = ref_audio
|
session["ref_audio"] = ref_audio
|
||||||
|
if instructions is not None:
|
||||||
|
session["instructions"] = instructions
|
||||||
|
if initial_user_text is not None:
|
||||||
|
extra_body = session["extra_body"]
|
||||||
|
assert isinstance(extra_body, dict)
|
||||||
|
extra_body["duplex_initial_user_text"] = initial_user_text
|
||||||
if session_id:
|
if session_id:
|
||||||
session["session_id"] = session_id
|
session["session_id"] = session_id
|
||||||
await self.send({"type": "session.update", "session": session})
|
await self.send({"type": "session.update", "session": session})
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ class MiniCPMO45NativeDuplexServingAdapter:
|
|||||||
"ref_audio_data",
|
"ref_audio_data",
|
||||||
"ref_audio_format",
|
"ref_audio_format",
|
||||||
"ref_audio_sample_rate_hz",
|
"ref_audio_sample_rate_hz",
|
||||||
|
"initial_user_text",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -90,6 +91,9 @@ class MiniCPMO45NativeDuplexServingAdapter:
|
|||||||
raise ValueError("ref_audio_path is not accepted by native duplex; use ref_audio URI instead")
|
raise ValueError("ref_audio_path is not accepted by native duplex; use ref_audio URI instead")
|
||||||
cls.validate_client_config(config)
|
cls.validate_client_config(config)
|
||||||
runtime_config: dict[str, object] = {"instructions": config.instructions}
|
runtime_config: dict[str, object] = {"instructions": config.instructions}
|
||||||
|
initial_user_text = extra_body.pop("duplex_initial_user_text", None)
|
||||||
|
if isinstance(initial_user_text, str) and initial_user_text:
|
||||||
|
runtime_config["initial_user_text"] = initial_user_text
|
||||||
cls._apply_default_scheduler_policy(runtime_config, config=config, model_config=model_config)
|
cls._apply_default_scheduler_policy(runtime_config, config=config, model_config=model_config)
|
||||||
|
|
||||||
ref_audio = config.ref_audio
|
ref_audio = config.ref_audio
|
||||||
@@ -108,6 +112,7 @@ class MiniCPMO45NativeDuplexServingAdapter:
|
|||||||
runtime_config,
|
runtime_config,
|
||||||
model_config=model_config,
|
model_config=model_config,
|
||||||
instructions=config.instructions,
|
instructions=config.instructions,
|
||||||
|
initial_user_text=initial_user_text,
|
||||||
ref_sample_count=None,
|
ref_sample_count=None,
|
||||||
)
|
)
|
||||||
config.extra_body = extra_body
|
config.extra_body = extra_body
|
||||||
@@ -130,6 +135,7 @@ class MiniCPMO45NativeDuplexServingAdapter:
|
|||||||
runtime_config,
|
runtime_config,
|
||||||
model_config=model_config,
|
model_config=model_config,
|
||||||
instructions=config.instructions,
|
instructions=config.instructions,
|
||||||
|
initial_user_text=initial_user_text,
|
||||||
ref_sample_count=len(wav_np),
|
ref_sample_count=len(wav_np),
|
||||||
)
|
)
|
||||||
config.extra_body = extra_body
|
config.extra_body = extra_body
|
||||||
@@ -167,6 +173,7 @@ class MiniCPMO45NativeDuplexServingAdapter:
|
|||||||
*,
|
*,
|
||||||
model_config: Any,
|
model_config: Any,
|
||||||
instructions: object,
|
instructions: object,
|
||||||
|
initial_user_text: object,
|
||||||
ref_sample_count: int | None,
|
ref_sample_count: int | None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Precompute the exact session-context token count for the engine.
|
"""Precompute the exact session-context token count for the engine.
|
||||||
@@ -186,6 +193,7 @@ class MiniCPMO45NativeDuplexServingAdapter:
|
|||||||
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
|
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
|
||||||
instructions,
|
instructions,
|
||||||
ref_sample_count is not None,
|
ref_sample_count is not None,
|
||||||
|
initial_user_text,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
prefix_ids = tokenizer.encode(prefix, add_special_tokens=False)
|
prefix_ids = tokenizer.encode(prefix, add_special_tokens=False)
|
||||||
|
|||||||
@@ -43,7 +43,11 @@ class MiniCPMO45DuplexPolicy:
|
|||||||
return max(0, int(sample_count) // cls.SAMPLES_PER_AUDIO_TOKEN)
|
return max(0, int(sample_count) // cls.SAMPLES_PER_AUDIO_TOKEN)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def session_context_texts(instructions: object, has_ref_audio: bool) -> tuple[str, str]:
|
def session_context_texts(
|
||||||
|
instructions: object,
|
||||||
|
has_ref_audio: bool,
|
||||||
|
initial_user_text: object = None,
|
||||||
|
) -> tuple[str, str]:
|
||||||
"""System-context prefix/suffix, matching MiniCPMODuplex.prepare()."""
|
"""System-context prefix/suffix, matching MiniCPMODuplex.prepare()."""
|
||||||
system_prompt = (
|
system_prompt = (
|
||||||
instructions if isinstance(instructions, str) and instructions else "Streaming Omni Conversation."
|
instructions if isinstance(instructions, str) and instructions else "Streaming Omni Conversation."
|
||||||
@@ -53,6 +57,8 @@ class MiniCPMO45DuplexPolicy:
|
|||||||
if has_ref_audio:
|
if has_ref_audio:
|
||||||
prefix += "\n<|audio_start|>"
|
prefix += "\n<|audio_start|>"
|
||||||
suffix = "<|audio_end|>" + suffix
|
suffix = "<|audio_end|>" + suffix
|
||||||
|
if isinstance(initial_user_text, str) and initial_user_text:
|
||||||
|
suffix += f"\n<|im_start|>user\n{initial_user_text}<|im_end|>\n<|im_start|>assistant\n"
|
||||||
return prefix, suffix
|
return prefix, suffix
|
||||||
|
|
||||||
SPECIAL_TOKEN_FIELDS: dict[str, str] = {
|
SPECIAL_TOKEN_FIELDS: dict[str, str] = {
|
||||||
|
|||||||
@@ -128,6 +128,7 @@ class MiniCPMO45Stage0DuplexRuntime:
|
|||||||
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
|
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
|
||||||
session_config.get("instructions"),
|
session_config.get("instructions"),
|
||||||
ref_audio is not None,
|
ref_audio is not None,
|
||||||
|
(runtime_config or {}).get("initial_user_text"),
|
||||||
)
|
)
|
||||||
for token_id in self._encode_text(prefix):
|
for token_id in self._encode_text(prefix):
|
||||||
state.context_embeds.append(self._embed_token(token_id))
|
state.context_embeds.append(self._embed_token(token_id))
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ class ChatFallbackProjectorMixin:
|
|||||||
if response_config.max_tokens is not None:
|
if response_config.max_tokens is not None:
|
||||||
kwargs["max_tokens"] = response_config.max_tokens
|
kwargs["max_tokens"] = response_config.max_tokens
|
||||||
model_extra = dict(response_config.extra_body)
|
model_extra = dict(response_config.extra_body)
|
||||||
|
model_extra.pop("minicpmo45_native_duplex", None)
|
||||||
tools = model_extra.pop("realtime_response_tools", model_extra.pop("realtime_tools", None))
|
tools = model_extra.pop("realtime_response_tools", model_extra.pop("realtime_tools", None))
|
||||||
tool_choice = model_extra.pop(
|
tool_choice = model_extra.pop(
|
||||||
"realtime_response_tool_choice",
|
"realtime_response_tool_choice",
|
||||||
|
|||||||
@@ -1172,6 +1172,7 @@ class OmniDuplexSessionHandler(
|
|||||||
return (
|
return (
|
||||||
session.capabilities.implementation_level == "model_native_duplex"
|
session.capabilities.implementation_level == "model_native_duplex"
|
||||||
and session.capabilities.supports_input_append
|
and session.capabilities.supports_input_append
|
||||||
|
and session.config.extra_body.get("minicpmo45_native_duplex") is not False
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
Reference in new issue
Block a user