From 7619ed6b821bfdb57035a716d8dc3962896bed75 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:17:57 +0800 Subject: [PATCH] [CI] Reducing mithril-h100-pool in ready&merge CI, optimize ready CI duration (#4354) Signed-off-by: [Your Name] Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> Signed-off-by: Hongsheng Liu Signed-off-by: [Your Name] <[Your Email]> Co-authored-by: Hongsheng Liu --- .buildkite/test-amd-merge.yml | 26 -- .buildkite/test-amd-ready.yaml | 15 +- .buildkite/test-merge.yml | 385 +----------------- .buildkite/test-nightly.yml | 169 +++----- .buildkite/test-ready.yml | 117 ++---- .buildkite/test-weekly.yml | 243 +++++++++++ docs/contributing/ci/CI_5levels.md | 29 +- .../text_to_speech/glm_tts/end2end.py | 2 +- pyproject.toml | 3 +- .../test_invalid_audio_speech.py | 87 +++- .../test_bagel_mooncake_connector.py | 1 - .../test_bagel_shared_memory_connector.py | 2 - .../test_audiox_model_expansion.py | 2 +- .../test_cosyvoice3_expansion.py | 2 +- .../test_covo_audio_expansion.py | 2 +- .../test_dreamid_omni_cache_dit.py | 2 +- .../test_dynin_omni_expansion.py | 12 +- .../test_ernie_image_expansion.py | 2 +- ...est_flux1_dev_autoround_w4a16_expansion.py | 4 +- .../test_flux1_kontext_expansion.py | 5 +- .../test_flux1_schnell_expansion.py | 4 +- ...t_glm_tts.py => test_glm_tts_expansion.py} | 18 +- ...v2.py => test_higgs_audio_v2_expansion.py} | 16 +- .../test_magi_human_expansion.py | 2 +- .../test_mammoth_moda2_expansion.py | 2 +- .../test_ming_flash_omni_expansion.py | 14 +- ...ming_tts.py => test_ming_tts_expansion.py} | 16 +- ...moss_tts.py => test_moss_tts_expansion.py} | 76 +--- .../test_moss_tts_nano_expansion.py | 2 +- .../test_moss_tts_realtime_expansion.py | 125 ++++++ .../offline_inference/test_moss_tts_v1_5.py | 113 ----- .../test_moss_tts_v1_5_expansion.py | 115 ++++++ .../test_omnivoice_expansion.py | 2 +- ..._qwen2_5_omni_autoround_w4a16_expansion.py | 11 +- .../test_qwen2_5_omni_expansion.py | 4 +- .../test_sensenova_u1_expansion.py | 2 +- ...=> test_sensenova_u1_img2img_expansion.py} | 8 +- ...> test_sensenova_u1_text2img_expansion.py} | 8 +- .../test_stable_audio_expansion.py | 2 +- .../e2e/offline_inference/test_step_audio2.py | 303 -------------- .../test_step_audio2_expansion.py | 96 +++++ .../e2e/offline_inference/test_voxcpm2_tts.py | 20 +- ...l_tts.py => test_voxtral_tts_expansion.py} | 32 +- .../online_serving/test_audiox_expansion.py | 2 +- .../test_cosyvoice3_tts_expansion.py | 5 +- .../test_covo_audio_expansion.py | 2 +- .../test_dreamzero_expansion.py | 2 +- .../test_dynin_omni_expansion.py | 2 +- .../test_ernie_image_expansion.py | 2 +- .../online_serving/test_flux2_expansion.py | 2 +- .../test_flux_2_dev_expansion.py | 2 +- .../test_flux_kontext_expansion.py | 2 +- ...t_glm_tts.py => test_glm_tts_expansion.py} | 33 +- .../test_hidream_i1_full_expansion.py | 3 +- ...v2.py => test_higgs_audio_v2_expansion.py} | 52 +-- .../e2e/online_serving/test_higgs_audio_v3.py | 118 +----- .../online_serving/test_hunyuan_video_15.py | 75 ---- .../test_hunyuan_video_15_expansion.py | 2 +- .../e2e/online_serving/test_longcat_image.py | 69 ---- .../online_serving/test_longcat_image_edit.py | 69 ---- .../test_longcat_image_edit_expansion.py | 13 +- .../test_longcat_image_expansion.py | 12 +- ..._audio.py => test_mimo_audio_expansion.py} | 82 ++-- ...ming_tts.py => test_ming_tts_expansion.py} | 10 +- .../online_serving/test_moss_tts_expansion.py | 22 +- .../test_moss_tts_nano_expansion.py | 4 +- .../online_serving/test_nextstep_expansion.py | 2 +- .../test_qwen2_5_omni_expansion.py | 4 +- .../test_qwen3_tts_customvoice.py | 9 +- tests/e2e/online_serving/test_sd3.py | 61 --- .../e2e/online_serving/test_sd3_expansion.py | 2 +- tests/e2e/online_serving/test_step_audio2.py | 368 ----------------- .../test_step_audio2_expansion.py | 132 ++++++ tests/e2e/online_serving/test_voxtral_tts.py | 180 -------- .../test_voxtral_tts_expansion.py | 163 ++++++++ .../test_openai_audio_list_endpoints.py | 4 +- .../test_qwen3_omni_realtime_websocket.py | 21 - .../openai_api/test_serving_speech_batch.py | 18 +- tests/entrypoints/test_async_omni.py | 91 +++-- tests/helpers/assertions.py | 115 +++++- tests/helpers/env.py | 1 + tests/helpers/runtime.py | 63 +-- tests/helpers/stage_config.py | 75 +++- .../step_audio2/test_step_audio2_thinker.py | 151 +++++++ 84 files changed, 1718 insertions(+), 2428 deletions(-) rename tests/e2e/offline_inference/{test_glm_tts.py => test_glm_tts_expansion.py} (90%) rename tests/e2e/offline_inference/{test_higgs_audio_v2.py => test_higgs_audio_v2_expansion.py} (96%) rename tests/e2e/offline_inference/{test_ming_tts.py => test_ming_tts_expansion.py} (97%) rename tests/e2e/offline_inference/{test_moss_tts.py => test_moss_tts_expansion.py} (67%) create mode 100644 tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py delete mode 100644 tests/e2e/offline_inference/test_moss_tts_v1_5.py create mode 100644 tests/e2e/offline_inference/test_moss_tts_v1_5_expansion.py rename tests/e2e/offline_inference/{test_sensenova_u1_img2img.py => test_sensenova_u1_img2img_expansion.py} (96%) rename tests/e2e/offline_inference/{test_sensenova_u1_text2img.py => test_sensenova_u1_text2img_expansion.py} (96%) delete mode 100644 tests/e2e/offline_inference/test_step_audio2.py create mode 100644 tests/e2e/offline_inference/test_step_audio2_expansion.py rename tests/e2e/offline_inference/{test_voxtral_tts.py => test_voxtral_tts_expansion.py} (82%) rename tests/e2e/online_serving/{test_glm_tts.py => test_glm_tts_expansion.py} (74%) rename tests/e2e/online_serving/{test_higgs_audio_v2.py => test_higgs_audio_v2_expansion.py} (89%) delete mode 100644 tests/e2e/online_serving/test_hunyuan_video_15.py delete mode 100644 tests/e2e/online_serving/test_longcat_image.py delete mode 100644 tests/e2e/online_serving/test_longcat_image_edit.py rename tests/e2e/online_serving/{test_mimo_audio.py => test_mimo_audio_expansion.py} (82%) rename tests/e2e/online_serving/{test_ming_tts.py => test_ming_tts_expansion.py} (92%) delete mode 100644 tests/e2e/online_serving/test_sd3.py delete mode 100644 tests/e2e/online_serving/test_step_audio2.py create mode 100644 tests/e2e/online_serving/test_step_audio2_expansion.py delete mode 100644 tests/e2e/online_serving/test_voxtral_tts.py create mode 100644 tests/e2e/online_serving/test_voxtral_tts_expansion.py create mode 100644 tests/model_executor/models/step_audio2/test_step_audio2_thinker.py diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/test-amd-merge.yml index b1dcb78d2..fb703cb45 100644 --- a/.buildkite/test-amd-merge.yml +++ b/.buildkite/test-amd-merge.yml @@ -179,32 +179,6 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model" ' - # It is the whisper ASR validation step that is slow, not the Voxtral TTS generation. - # whisper ASR runs on CPU. - - label: "Voxtral-TTS E2E Test (Online Serving)" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 40m bash -c ' - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" - ' - - - label: "Voxtral-TTS E2E Test (Offline Inference)" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 30m bash -c ' - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" - ' - # TODO: Bagel test on ROCm is very unstable. @tjtanaa # Need to debug before reneable numerical changes across large PRs # # split Bagel Model Test with H100 (Real Weights) into three tests diff --git a/.buildkite/test-amd-ready.yaml b/.buildkite/test-amd-ready.yaml index 1da5edd19..8c645fff0 100644 --- a/.buildkite/test-amd-ready.yaml +++ b/.buildkite/test-amd-ready.yaml @@ -161,19 +161,6 @@ steps: - group: ":card_index_dividers: E2E Test" steps: - - label: "MiMo-Audio E2E Test with H100" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 30m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model" - ' - - label: "Qwen3-TTS E2E Test" agent_pool: mi325_1 depends_on: amd-build @@ -193,7 +180,7 @@ steps: commands: - | timeout 20m bash -c ' - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model" + pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model" ' # TODO: Bagel test on ROCm is very unstable. @tjtanaa diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index 96ca753e2..6af95f20f 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -61,13 +61,14 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Distributed Test" + depends_on: upload-merge-pipeline timeout_in_minutes: 60 commands: - | timeout 55m bash -c " set -e export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/distributed -m 'advanced_model and cuda' --run-level 'advanced_model' + pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model' " agents: queue: "mithril-h100-pool" @@ -164,28 +165,6 @@ steps: - group: ":card_index_dividers: E2E Test" depends_on: upload-merge-pipeline steps: - - label: "TTS · Step-Audio2 E2E Test" - timeout_in_minutes: 20 - commands: - - | - timeout 20m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_step_audio2.py -m 'advanced_model' --run-level 'advanced_model' - " - agents: - queue: "gpu_4_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - label: "TTS · Qwen3-TTS CustomVoice Test" source_file_dependencies: - tests/e2e/online_serving/test_qwen3_tts_customvoice.py @@ -240,19 +219,19 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "TTS · Ming-Omni-TTS Test" + - label: "TTS · Higgs-Audio-V3 Test" source_file_dependencies: - - tests/e2e/online_serving/test_ming_tts.py - - tests/e2e/offline_inference/test_ming_tts.py - - vllm_omni/model_executor/models/ming_tts/ - - vllm_omni/model_executor/stage_input_processors/ming_tts.py - - vllm_omni/deploy/ming_tts.yaml - timeout_in_minutes: 20 + - tests/e2e/online_serving/test_higgs_audio_v3.py + - vllm_omni/model_executor/models/higgs_audio_v3/ + - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py + - vllm_omni/deploy/higgs_multimodal_qwen3.yaml commands: - | timeout 20m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - pytest -s -v tests/e2e/online_serving/test_ming_tts.py tests/e2e/offline_inference/test_ming_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' + export VLLM_WORKER_MULTIPROC_METHOD=spawn + export VLLM_USE_DEEP_GEMM=0 + export VLLM_MOE_USE_DEEP_GEMM=0 + pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model' " agents: queue: "gpu_1_queue" @@ -264,105 +243,10 @@ steps: shm-size: "8gb" environment: - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "TTS · Voxtral-TTS Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_voxtral_tts.py - - tests/e2e/online_serving/test_voxtral_tts.py - - vllm_omni/model_executor/models/voxtral_tts/ - - vllm_omni/model_executor/stage_input_processors/voxtral_tts.py - - vllm_omni/deploy/voxtral_tts.yaml - timeout_in_minutes: 20 - commands: - - | - timeout 20m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py tests/e2e/online_serving/test_voxtral_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - - label: "TTS · GLM-TTS Test" - source_file_dependencies: - - tests/e2e/online_serving/test_glm_tts.py - - tests/e2e/offline_inference/test_glm_tts.py - - vllm_omni/model_executor/models/glm_tts/ - - vllm_omni/model_executor/stage_input_processors/glm_tts.py - - vllm_omni/deploy/glm_tts.yaml - timeout_in_minutes: 20 - commands: - - | - timeout 20m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - pytest -s -v tests/e2e/online_serving/test_glm_tts.py tests/e2e/offline_inference/test_glm_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - label: "TTS · IndexTTS2 Test" source_file_dependencies: - tests/e2e/online_serving/test_indextts2.py @@ -443,49 +327,6 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: "Omni · Mimo Audio Test" - source_file_dependencies: - - tests/e2e/online_serving/test_mimo_audio.py - - vllm_omni/model_executor/models/mimo_audio/ - - vllm_omni/model_executor/stage_input_processors/mimo_audio.py - - vllm_omni/deploy/mimo_audio.yaml - timeout_in_minutes: 30 - commands: - - pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m 'advanced_model and cuda' --run-level 'advanced_model' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - label: "TTS · VoxCPM2 Test" source_file_dependencies: - tests/e2e/offline_inference/test_voxcpm2_tts.py @@ -514,37 +355,6 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Omni · Higgs-Audio V2 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_higgs_audio_v2.py - - tests/e2e/offline_inference/test_higgs_audio_v2.py - - vllm_omni/model_executor/models/higgs_audio_v2/ - - vllm_omni/model_executor/stage_input_processors/higgs_audio_v2.py - - vllm_omni/deploy/higgs_audio_v2.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_USE_DEEP_GEMM=0 - export VLLM_MOE_USE_DEEP_GEMM=0 - pytest -s -v tests/e2e/online_serving/test_higgs_audio_v2.py tests/e2e/offline_inference/test_higgs_audio_v2.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - label: "TTS · SoulX-Singer Test" source_file_dependencies: - tests/e2e/offline_inference/test_soulxsinger.py @@ -614,88 +424,6 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: "Diffusion · Longcat Image Edit Test" - source_file_dependencies: - - tests/e2e/online_serving/test_longcat_image_edit.py - - vllm_omni/diffusion/models/longcat_image/ - timeout_in_minutes: 20 - commands: - - pytest -s -v tests/e2e/online_serving/test_longcat_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - - label: "Diffusion · Longcat Image Test" - source_file_dependencies: - - tests/e2e/online_serving/test_longcat_image.py - - vllm_omni/diffusion/models/longcat_image/ - timeout_in_minutes: 20 - commands: - - pytest -s -v tests/e2e/online_serving/test_longcat_image.py -m 'advanced_model and cuda' --run-level 'advanced_model' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - label: "Diffusion · Bagel Test" source_file_dependencies: - tests/e2e/offline_inference/test_bagel.py @@ -875,50 +603,6 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: "Diffusion · HunyuanVideo 1.5 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_hunyuan_video_15.py - - vllm_omni/diffusion/models/hunyuan_video/ - timeout_in_minutes: 40 - commands: - - | - timeout 40m bash -c " - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - label: "Diffusion · Z Image Test" source_file_dependencies: - tests/e2e/offline_inference/test_z_image.py @@ -960,48 +644,3 @@ steps: - "HF_TOKEN" volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - - label: "Diffusion · SD3 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_sd3.py - - vllm_omni/diffusion/models/sd3/ - timeout_in_minutes: 45 - commands: - - | - timeout 45m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_sd3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 6db22a6a6..0569eb943 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -49,25 +49,6 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: ":full_moon: Omni · Function Test with L4" - timeout_in_minutes: 90 - commands: - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model" - agents: - queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - label: ":full_moon: Omni · Doc Test with L4" timeout_in_minutes: 90 commands: @@ -398,6 +379,8 @@ steps: timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py agents: queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU @@ -568,54 +551,52 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test" - key: nightly-tts-performance-higgs-audio-v3 - depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_USE_DEEP_GEMM="0" - - export VLLM_MOE_USE_DEEP_GEMM="0" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: + - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test" + key: nightly-tts-performance-higgs-audio-v3 + timeout_in_minutes: 180 + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: - name: devshm - mountPath: /dev/shm + emptyDir: + medium: Memory - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate # Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below. - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" @@ -636,11 +617,7 @@ steps: tests/e2e/online_serving/test_qwen_image_expansion.py tests/e2e/online_serving/test_qwen_image_edit_expansion.py tests/e2e/online_serving/test_qwen_image_layered_expansion.py - tests/e2e/online_serving/test_longcat_image_expansion.py - tests/e2e/online_serving/test_longcat_image_edit_expansion.py - tests/e2e/online_serving/test_flux_2_dev_expansion.py tests/e2e/online_serving/test_bagel_expansion.py - tests/e2e/offline_inference/test_sensenova_u1_expansion.py -m "full_model and diffusion and H100 and not distributed_cuda" --run-level "full_model" agents: @@ -726,10 +703,6 @@ steps: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_layered_expansion.py - tests/e2e/online_serving/test_longcat_image_expansion.py - tests/e2e/online_serving/test_longcat_image_edit_expansion.py - tests/e2e/online_serving/test_flux_2_dev_expansion.py - tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" agents: @@ -808,13 +781,14 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU DreamZero" + - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100" timeout_in_minutes: 120 commands: - >- pytest -sv - tests/e2e/online_serving/test_dreamzero_expansion.py - -m "full_model and diffusion and H100 and distributed_cuda" + tests/e2e/online_serving/test_glm_image_expansion.py + tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py + -m "full_model and diffusion and H100" --run-level "full_model" agents: queue: "mithril-h100-pool" @@ -851,45 +825,6 @@ steps: type: DirectoryOrCreate - - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100" - timeout_in_minutes: 120 - commands: - - pytest -sv tests/e2e/online_serving/test_glm_image_expansion.py -m "full_model and diffusion and H100" --run-level "full_model" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" timeout_in_minutes: 60 commands: @@ -1356,7 +1291,7 @@ steps: timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" - - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model" + - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: @@ -1580,7 +1515,7 @@ steps: key: nightly-testcase-statistics timeout_in_minutes: 120 depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" + if: build.env("NIGHTLY") == "1" commands: - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index a0f40d05a..d12267fe0 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -184,96 +184,43 @@ steps: commands: - | timeout 20m bash -c " - pytest -sv tests/distributed/ -m 'core_model and cuda' --run-level 'core_model' + pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model' " agents: - queue: "mithril-h100-pool" + queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" - - group: ":card_index_dividers: Entrypoints Test" + - label: "Entrypoints Test" depends_on: upload-ready-pipeline - steps: - - label: "Entrypoints Test with H100" - commands: - - | - timeout 30m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -sv tests/entrypoints/ -m 'core_model and cuda and H100' --run-level 'core_model' - " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - - label: "Entrypoints Test with L4" - commands: - - | - timeout 30m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -sv tests/entrypoints/ -m 'core_model and cuda and L4' --run-level 'core_model' - " - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + commands: + - | + timeout 30m bash -c " + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model' + " + agents: + queue: "gpu_1_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" - group: ":card_index_dividers: E2E Test" depends_on: upload-ready-pipeline diff --git a/.buildkite/test-weekly.yml b/.buildkite/test-weekly.yml index aefb424e5..d71bdae0a 100644 --- a/.buildkite/test-weekly.yml +++ b/.buildkite/test-weekly.yml @@ -206,3 +206,246 @@ steps: - "HF_TOKEN" volumes: - "/fsx/hf_cache:/fsx/hf_cache" + + - group: ":card_index_dividers: E2E Tests" + depends_on: upload-weekly-pipeline + if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" + steps: + - label: "Omni · L4" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model" + - >- + pytest -s -v tests/e2e/ + --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py + --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py + -m "slow and L4 and omni" + --run-level "full_model" + agents: + queue: "gpu_4_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + + - label: "Omni · H100" + timeout_in_minutes: 90 + commands: + - >- + pytest -sv + tests/e2e/ -m "slow and H100 and omni" + --run-level "full_model" + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + - label: "TTS · L4" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - | + failed=0 + for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do + pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 + done + exit $$failed + agents: + queue: "gpu_1_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + + - label: "TTS · H100" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - >- + pytest -sv + tests/e2e/ -m "slow and H100 and tts" + --run-level "full_model" + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + - label: "Diffusion · L4" + timeout_in_minutes: 60 + commands: + - >- + pytest -sv + tests/e2e/ -m "slow and diffusion and L4" + --run-level "full_model" + agents: + queue: "gpu_4_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + + - label: "Diffusion · H100 · Single-GPU" + timeout_in_minutes: 120 + commands: + - >- + pytest -sv + tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda" + --run-level "full_model" + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + - label: "Diffusion · H100 · Multi-GPU" + timeout_in_minutes: 90 + commands: + - >- + pytest -sv + tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda" + --run-level "full_model" + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index c27f1fc9a..17b4a3792 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -67,7 +67,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L2
(E2E across models & GPU-required UT) Online (basic deployment scenarios):
dummy, normal inference function (output format, stream), some instance startup UT - High-priority models + online basic scenarios + request success validation + High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy) High-priority features (using random lightweight models) High-priority interfaces (using random lightweight models) core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion @@ -89,7 +89,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L3
(Important Perf & Integration & Accuracy) Online & Offline (multiple deployment scenarios):
real model, normal inference function, normal accuracy - High/medium-priority models with real weights + online/offline key scenarios + basic accuracy validation + High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy Medium-priority features (using random lightweight models) Medium-priority interfaces (using random lightweight models) advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion @@ -113,9 +113,9 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L4
(Perf & Integration & Accuracy) Online: full functional scenarios + performance test + doc test + accuracy test - High-priority models: function, performance, accuracy, and doc testing
Medium/low-priority models: function and doc testing - Low-priority features (using random lightweight models) - Low-priority interfaces (using random lightweight models) + High-priority models: function, performance, accuracy, and doc testing
Medium-priority models: function and doc testing + Low-priority features (using real weights) + Low-priority interfaces (using real weights) full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion <3 hour @@ -144,7 +144,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L5
(Stability & Reliability) Online: long-term stability test + reliability test - Long-term stability and reliability testing for high-priority models + Long-term stability and reliability testing for high-priority models
Low-priority models: function and doc testing / Invalid-parameter validation for high-priority interfaces slow and hardware_test(H100, L4, etc.) and omni/tts/diffusion @@ -446,7 +446,11 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that - ***Time Cost***: Execution time is controlled within ***15 minutes*** to ensure fast feedback. - ***L2 (Basic End-to-End Testing)***: - - ***Scope***: Covers two basic deployment scenarios: `online` (serving) and `offline` (inference). - - ***Focus***: Uses `dummy` models or lightweight real models to verify that the entire chain from request input to result output works normally, including output data structure, streaming (stream) support, etc. Also includes some unit tests that require launching independent service instances. + - ***Focus***: Uses `dummy` weights (via deploy YAML patching at `core_model`) or lightweight real models to verify that the entire chain from request input to result output works normally, including output data structure, streaming (stream) support, and **cheap payload checks** at `--run-level core_model` (see below). Also includes some unit tests that require launching independent service instances. + - ***L2 response validation (`core_model`)***: Implemented in `tests/helpers/assertions.py` and invoked by `OpenAIClientHandler` based on `--run-level`. At L2 we require **request success** plus minimal output sanity—not full accuracy: + - **Speech / TTS** (`assert_audio_speech_response`): decoded audio must be present and non-empty (or exceed `min_audio_bytes` when set in `request_config`); `response_format` must match the returned content-type (e.g. `wav`, `pcm`). Whisper transcript similarity, PCM HNR, and preset-voice gender checks run only at L3+. + - **Diffusion** (`assert_diffusion_response`): at least one non-empty image, video, or audio artifact. Resolution/frame-count parity and other deep checks run only at L3+. + - **Omni multimodal** (`assert_omni_response`): L2 asserts successful completion; keyword, transcript, and cross-modal similarity checks run only at L3+. - ***Characteristic***: Requires ***GPU*** resources to perform model computations. ### 1.3 Test Directory and Execution Files @@ -555,6 +559,11 @@ L3 level testing executes after code is merged into the main branch. Its core pu - 1. ***Inference Functionality***: Ensures real models can perform forward computation normally and return results. 2. ***Accuracy Compliance***: Verifies that the model's evaluation metrics (e.g., accuracy) meet the expected baseline, preventing code changes from introducing accuracy issues. 3. ***Important Performance***: Verifies whether performance (e.g., P99 latency, throughput) in core scenarios meets preset thresholds. +- ***L3 response validation (`advanced_model`)***: At `--run-level advanced_model`, `tests/helpers/assertions.py` adds semantic checks on top of L2 payload gates (see Chapter 1): + - **Speech / TTS** (`assert_audio_speech_response`): Whisper transcript of returned audio vs `request_config["input"]` (cosine similarity > 0.9 when input text is set); optional `min_audio_bytes` floor; PCM harmonic-to-noise ratio when `response_format` is `pcm`; **preset voice gender** via `_assert_preset_voice_gender_from_audio` when `voice` matches a known preset in `_PRESET_VOICE_GENDER_MAP` (pitch/F0-based classifier; skipped for unknown voices or `pcm` output). + - **Omni multimodal** (`assert_omni_response`): non-empty text/audio outputs; `key_words` in transcript or text; text–audio similarity / containment; preset **`speaker`** gender check (same helper as TTS). + - **Diffusion** (`assert_*_diffusion_response`): image/video dimension and frame-count parity with request parameters where configured. + - **Accuracy suites**: pixel/video similarity and metric baselines under `/tests/e2e/accuracy/` (separate from inline helper assertions). ### 2.3 Test Directory and Execution Files @@ -582,9 +591,9 @@ L3 level testing executes after code is merged into the main branch. Its core pu **Explanation**: - @pytest.mark.advanced_model: Marks the test as L3 merge level, indicating deep validation with real models. @pytest.mark.full_model: Marks L4 nightly-only suites (e.g. `test_*_expansion.py`, doc examples). + @pytest.mark.advanced_model: Marks the test as L3 merge level (`--run-level advanced_model`): real weights, Whisper/similarity/keyword checks, diffusion deep checks, and **preset voice/speaker gender** validation where applicable. @pytest.mark.full_model: Marks L4 nightly-only suites (e.g. `test_*_expansion.py`, doc examples). - @pytest.mark.core_model: Marks the test as L1 or L2 level, indicating that this test case validates the basic functionality of the core model. It uses mock weights and only checks if the relevant interface functions correctly. + @pytest.mark.core_model: Marks the test as L1 or L2 level. At `--run-level core_model`, validation is limited to request success plus cheap payload checks (e.g. non-empty audio bytes, response format/content-type, non-empty diffusion outputs)—not Whisper, keyword, or accuracy gates. Deploy YAML may use `load_format: dummy` for fast PR feedback. @pytest.mark.parametrize: A parameterization decorator that allows abstracting test data into parameters, enabling reuse of the same test logic across different data configurations. indirect=True indicates that parameters will be passed to the fixture for processing. @@ -680,7 +689,7 @@ L3 level testing executes after code is merged into the main branch. Its core pu **Single Request**: The comment clearly states this is a single-request completion test. For concurrent testing, it can be extended to multiple requests using request_num = n. - **Implicit Validation**: The `send_omni_request` and `send_diffusion_request` methods internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model and full_model perform deep validation. + **Implicit Validation**: `send_omni_request`, `send_audio_speech_request`, and `send_diffusion_request` call `assert_*_response` helpers with the session `--run-level`. At `core_model`, checks are limited to success plus cheap payload sanity (non-empty audio/media, format/content-type, optional `min_audio_bytes`). At `advanced_model` and `full_model`, deep validation adds Whisper transcripts, keyword/similarity, diffusion dimension checks, PCM HNR, etc. **Audio output debugging**: Deep validation may transcribe returned audio via `convert_audio_bytes_to_text` (Whisper). If an audio keyword or text–audio similarity assertion fails, set `VLLM_OMNI_KEEP_REQUEST_MEDIA=1` before running pytest to keep the intermediate WAV files for inspection (see [Test helper environment variables](#test-helper-environment-variables)). diff --git a/examples/offline_inference/text_to_speech/glm_tts/end2end.py b/examples/offline_inference/text_to_speech/glm_tts/end2end.py index d1f19f8a5..6624c2b35 100644 --- a/examples/offline_inference/text_to_speech/glm_tts/end2end.py +++ b/examples/offline_inference/text_to_speech/glm_tts/end2end.py @@ -101,7 +101,7 @@ def _extract_sample_rate(audio_mm: dict) -> int: def _modify_deploy_config(base_path: str, async_chunk: bool) -> str: """Build deploy config with explicit sync/async mode and eager execution. - Mirrors the logic in ``tests/e2e/offline_inference/test_glm_tts.py`` + Mirrors the logic in ``tests/e2e/offline_inference/test_glm_tts_expansion.py`` (``_get_deploy_config``) so that example runs match CI behavior. """ with open(base_path) as f: diff --git a/pyproject.toml b/pyproject.toml index 962f58a8f..b4a3fef5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,8 @@ dev = [ "torchdiffeq>=0.2.5", # for covo_audio Test "FlagEmbedding", # for hunyuan_image3 tests "kernels<0.15.0", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0) - "s3tokenizer>=0.3.0" # for tts test + "s3tokenizer>=0.3.0", # for tts test + "step-audio2>=1.0.0", # Step-Audio2 Token2Wav (flashcosyvoice / hyperpyyaml) ] demo = [ diff --git a/tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py b/tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py index 2c2e30c65..0de862ae0 100644 --- a/tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py +++ b/tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""HTTP + WebSocket validation for Qwen3-TTS Base: ``/v1/audio/speech``, ``/v1/audio/speech/stream``, batch, voices.""" +"""HTTP + WebSocket validation for Qwen3-TTS Base and Higgs Audio V3 ``/v1/audio/speech`` (and related routes).""" from __future__ import annotations @@ -57,6 +57,19 @@ _QWEN3_TTS_SPEECH = [ ), ] +_HIGGS_AUDIO_V3_SPEECH = [ + pytest.param( + OmniServerParams( + model="bosonai/higgs-audio-v3-tts-4b", + stage_config_path=get_deploy_config_path("higgs_multimodal_qwen3.yaml"), + server_args=_SPEECH_SERVER_ARGS, + env_dict={"VLLM_USE_DEEP_GEMM": "0", "VLLM_MOE_USE_DEEP_GEMM": "0"}, + ), + id="higgs_audio_v3", + marks=_L4_SPEECH_HW, + ), +] + def _apply_batch_overrides(body: dict[str, object], *, loc: str, overrides: dict[str, object]) -> None: if loc == "batch": @@ -240,6 +253,78 @@ def test_speech_invalid_field_values( ) +# Higgs Audio V3: ``references[]`` cookbook alias and serving-layer 400 rejections. +# Raw HTTP keeps JSON bodies intact (OpenAI SDK strips unknown fields such as ``references``). +@pytest.mark.parametrize( + "extra_json, err_message", + [ + pytest.param({"input": "", "response_format": "wav"}, ("input", "empty"), id="input_empty"), + pytest.param({"input": " ", "response_format": "wav"}, ("input", "empty"), id="input_whitespace_only"), + pytest.param( + {"input": "Hello world.", "max_new_tokens": 0}, ("max_new_tokens", "least 1"), id="max_new_tokens_below_min" + ), + pytest.param( + { + "input": "Hello world.", + "references": [ + {"audio_path": REF_AUDIO_URL, "text": REF_TEXT}, + {"audio_path": REF_AUDIO_URL, "text": REF_TEXT}, + ], + "response_format": "wav", + }, + ("references", "only supports a single reference"), + id="multi_reference_payload", + ), + pytest.param( + {"input": "Hello world.", "references": [], "response_format": "wav"}, + ("references", "non-empty"), + id="references_empty_array", + ), + pytest.param( + {"input": "Hello world.", "references": [{"text": REF_TEXT}], "response_format": "wav"}, + ("audio_path", "required"), + id="references_missing_audio_path", + ), + pytest.param( + {"input": "Hello world.", "references": [1], "response_format": "wav"}, + ("references[0]", "must be an object"), + id="references_bad_element_type", + ), + pytest.param( + { + "input": "Hello world.", + "ref_audio": REF_AUDIO_URL, + "references": [{"audio_path": "https://example.com/other.wav"}], + "response_format": "wav", + }, + ("mutually exclusive", "ref_audio", "references"), + id="ref_audio_and_references_exclusive", + ), + pytest.param( + { + "input": "Hello world.", + "ref_text": "different transcript", + "references": [{"audio_path": REF_AUDIO_URL, "text": REF_TEXT}], + "response_format": "wav", + }, + ("references[0].text", "ref_text", "conflict"), + id="references_text_conflicts_ref_text", + ), + ], +) +@pytest.mark.parametrize("omni_server", _HIGGS_AUDIO_V3_SPEECH, indirect=True) +def test_speech_higgs_invalid_field_values( + omni_server: OmniServer, + openai_client: OpenAIClientHandler, + extra_json: dict[str, object], + err_message: str | tuple[str, ...], +) -> None: + body = {"model": omni_server.model, **extra_json} + openai_client.send_audio_speech_http_request( + {"json": body, "timeout": 120, "err_code": 400, "err_message": err_message} + ) + + # ───────────────────────────────────────────────────────────────────────────── # POST /v1/audio/speech/batch (JSON) # ───────────────────────────────────────────────────────────────────────────── diff --git a/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py b/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py index cac7d7ed0..fef63f4f8 100644 --- a/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py +++ b/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py @@ -266,7 +266,6 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str: return temp_file.name -@pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) diff --git a/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py b/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py index c133dd56c..c21af394a 100644 --- a/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py +++ b/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py @@ -227,7 +227,6 @@ def _resolve_deploy_config(config_path: str, run_level: str) -> str: return config_path -@pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.diffusion @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) @@ -244,7 +243,6 @@ def test_bagel_img2img_shared_memory_connector(run_level): _validate_pixels(generated_image) -@pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.diffusion @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) diff --git a/tests/e2e/offline_inference/test_audiox_model_expansion.py b/tests/e2e/offline_inference/test_audiox_model_expansion.py index b78dc1a5e..d4b2f5223 100644 --- a/tests/e2e/offline_inference/test_audiox_model_expansion.py +++ b/tests/e2e/offline_inference/test_audiox_model_expansion.py @@ -21,7 +21,7 @@ models = [os.environ.get("AUDIOX_TEST_MODEL", "zhangj1an/audiox_random")] _OMNI_RUNNER_PARAMS = [(m, None, {"model_class_name": "AudioXPipeline"}) for m in models] pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True, ids=models), ] diff --git a/tests/e2e/offline_inference/test_cosyvoice3_expansion.py b/tests/e2e/offline_inference/test_cosyvoice3_expansion.py index ea6ac8dd3..49079266d 100644 --- a/tests/e2e/offline_inference/test_cosyvoice3_expansion.py +++ b/tests/e2e/offline_inference/test_cosyvoice3_expansion.py @@ -141,7 +141,7 @@ _OMNI_RUNNER_PARAMS = [ ), ] pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.tts, pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True), ] diff --git a/tests/e2e/offline_inference/test_covo_audio_expansion.py b/tests/e2e/offline_inference/test_covo_audio_expansion.py index 3c6e5efb7..ca05932ef 100644 --- a/tests/e2e/offline_inference/test_covo_audio_expansion.py +++ b/tests/e2e/offline_inference/test_covo_audio_expansion.py @@ -21,7 +21,7 @@ def get_question(): return "请回答这段音频里的问题。" -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1}) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) diff --git a/tests/e2e/offline_inference/test_dreamid_omni_cache_dit.py b/tests/e2e/offline_inference/test_dreamid_omni_cache_dit.py index a3f57064f..e22aa1174 100644 --- a/tests/e2e/offline_inference/test_dreamid_omni_cache_dit.py +++ b/tests/e2e/offline_inference/test_dreamid_omni_cache_dit.py @@ -30,7 +30,7 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion, pytest.mark.cache, pytest.mark.slow] +pytestmark = [pytest.mark.diffusion, pytest.mark.cache, pytest.mark.slow] REPO_ROOT = Path(__file__).resolve().parents[3] DEFAULT_MODEL_PATH = REPO_ROOT / "dreamid_omni" diff --git a/tests/e2e/offline_inference/test_dynin_omni_expansion.py b/tests/e2e/offline_inference/test_dynin_omni_expansion.py index 7aae111f0..13b9e5cb3 100644 --- a/tests/e2e/offline_inference/test_dynin_omni_expansion.py +++ b/tests/e2e/offline_inference/test_dynin_omni_expansion.py @@ -34,7 +34,7 @@ test_params = [(model, stage_config) for model in models for stage_config in sta DYNIN_CONFIG_PATH = str(_DEFAULT_DYNIN_CONFIG_PATH) if _DEFAULT_DYNIN_CONFIG_PATH is not None else None pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.omni, pytest.mark.parametrize("omni_runner", test_params, indirect=True), ] @@ -290,7 +290,7 @@ def _numel(value: Any) -> int: return 0 -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}) +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}) def test_dynin_t2i_decode_to_image(omni_runner) -> None: _configure_dynin_config_env() prompt = _build_t2i_decode_prompt(dynin_config_path=DYNIN_CONFIG_PATH) @@ -304,7 +304,7 @@ def test_dynin_t2i_decode_to_image(omni_runner) -> None: assert _numel(image_value) > 0 -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}) +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}) def test_dynin_mmu_to_text(omni_runner) -> None: _configure_dynin_config_env() tokenizer = AutoTokenizer.from_pretrained(omni_runner.model_name, trust_remote_code=True) @@ -322,7 +322,7 @@ def test_dynin_mmu_to_text(omni_runner) -> None: assert text_content -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}) +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}) def test_dynin_image_to_text(omni_runner) -> None: _configure_dynin_config_env() tokenizer = AutoTokenizer.from_pretrained(omni_runner.model_name, trust_remote_code=True) @@ -341,7 +341,7 @@ def test_dynin_image_to_text(omni_runner) -> None: assert text_content -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}) +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}) def test_dynin_speech_to_text(omni_runner) -> None: _configure_dynin_config_env() tokenizer = AutoTokenizer.from_pretrained(omni_runner.model_name, trust_remote_code=True) @@ -360,7 +360,7 @@ def test_dynin_speech_to_text(omni_runner) -> None: assert text_content -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}) +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}) def test_dynin_t2s_decode_to_audio(omni_runner) -> None: _configure_dynin_config_env() prompt = _build_t2s_decode_prompt(dynin_config_path=DYNIN_CONFIG_PATH) diff --git a/tests/e2e/offline_inference/test_ernie_image_expansion.py b/tests/e2e/offline_inference/test_ernie_image_expansion.py index 2375ab38b..19793ff00 100644 --- a/tests/e2e/offline_inference/test_ernie_image_expansion.py +++ b/tests/e2e/offline_inference/test_ernie_image_expansion.py @@ -30,7 +30,7 @@ _OMNI_RUNNER_PARAM = ( ) pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/offline_inference/test_flux1_dev_autoround_w4a16_expansion.py b/tests/e2e/offline_inference/test_flux1_dev_autoround_w4a16_expansion.py index a9b24ef9d..4e9e30712 100644 --- a/tests/e2e/offline_inference/test_flux1_dev_autoround_w4a16_expansion.py +++ b/tests/e2e/offline_inference/test_flux1_dev_autoround_w4a16_expansion.py @@ -105,7 +105,7 @@ def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]: @pytest.mark.skip(reason="BF16 baseline model cannot be loaded correctly yet.") -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}) @pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True) @@ -124,7 +124,7 @@ def test_flux_autoround_w4a16_generates_image(omni_runner: OmniRunner): @pytest.mark.skip(reason="BF16 baseline model cannot be loaded correctly yet.") -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}) def test_flux_autoround_w4a16_memory_savings(): diff --git a/tests/e2e/offline_inference/test_flux1_kontext_expansion.py b/tests/e2e/offline_inference/test_flux1_kontext_expansion.py index 3ab1aa401..abf7bbc2c 100644 --- a/tests/e2e/offline_inference/test_flux1_kontext_expansion.py +++ b/tests/e2e/offline_inference/test_flux1_kontext_expansion.py @@ -14,6 +14,7 @@ from __future__ import annotations import pytest from vllm.assets.image import ImageAsset +from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunnerHandler from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -30,7 +31,7 @@ _OMNI_RUNNER_PARAM = ( ) pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] @@ -45,6 +46,7 @@ def _sampling_512() -> OmniDiffusionSamplingParams: ) +@hardware_test(res={"cuda": "H100"}, num_cards=2) def test_flux_kontext_text_to_image(omni_runner_handler: OmniRunnerHandler): """Test FluxKontext text-to-image generation with real model.""" request_config = { @@ -55,6 +57,7 @@ def test_flux_kontext_text_to_image(omni_runner_handler: OmniRunnerHandler): omni_runner_handler.send_diffusion_request(request_config) +@hardware_test(res={"cuda": "H100"}, num_cards=2) def test_flux_kontext_image_edit(omni_runner_handler: OmniRunnerHandler): """Test FluxKontext image-to-image editing with real model.""" input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB") diff --git a/tests/e2e/offline_inference/test_flux1_schnell_expansion.py b/tests/e2e/offline_inference/test_flux1_schnell_expansion.py index 583ceaf59..de6a758f6 100644 --- a/tests/e2e/offline_inference/test_flux1_schnell_expansion.py +++ b/tests/e2e/offline_inference/test_flux1_schnell_expansion.py @@ -5,6 +5,7 @@ import pytest +from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunnerHandler from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -13,12 +14,13 @@ MODEL = "black-forest-labs/FLUX.1-schnell" _OMNI_RUNNER_PARAM = (MODEL, None) pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] +@hardware_test(res={"cuda": "H100"}, num_cards=1) def test_flux_schnell_text_to_image(omni_runner_handler: OmniRunnerHandler): """Test FLUX.1-schnell text-to-image generation.""" request_config = { diff --git a/tests/e2e/offline_inference/test_glm_tts.py b/tests/e2e/offline_inference/test_glm_tts_expansion.py similarity index 90% rename from tests/e2e/offline_inference/test_glm_tts.py rename to tests/e2e/offline_inference/test_glm_tts_expansion.py index 0b902c6bc..d6a8729f0 100644 --- a/tests/e2e/offline_inference/test_glm_tts.py +++ b/tests/e2e/offline_inference/test_glm_tts_expansion.py @@ -1,10 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""E2E Offline inference tests for GLM-TTS voice cloning. - -GLM-TTS is a zero-shot voice cloning model that always requires ref_audio. -There is no text-only / non-clone inference path in the official implementation. -""" +"""E2E expansion tests for GLM-TTS voice cloning offline inference (nightly CI).""" import os import tempfile @@ -23,11 +19,12 @@ from tests.helpers.runtime import OmniResponse from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config from vllm_omni.model_executor.models.glm_tts.glm_tts import build_glm_tts_prefill_metadata +pytestmark = [pytest.mark.slow, pytest.mark.tts] + MODEL = os.environ.get("GLM_TTS_MODEL_PATH", "zai-org/GLM-TTS") REF_TEXT = "他当时还跟线下其他的站姐吵架，然后，打架进局子了。" DEPLOY_CONFIG = get_deploy_config_path("glm_tts.yaml") - REFERENCE_PROMPT_WAV_PATH = get_asset_path("glm_tts/jiayan_zh.wav") ASYNC_CHUNK_MODES = [ @@ -84,9 +81,7 @@ def _get_deploy_config(*, async_chunk: bool) -> str: ) -@pytest.mark.advanced_model -@pytest.mark.omni -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("async_chunk", ASYNC_CHUNK_MODES) def test_offline_voice_clone_zh(async_chunk: bool) -> None: """ @@ -94,9 +89,6 @@ def test_offline_voice_clone_zh(async_chunk: bool) -> None: Deploy Setting: glm_tts.yaml with explicit sync/async mode, enforce_eager Input Modal: text + ref_audio + ref_text Output Modal: audio - - Uses the official jiayan_zh.wav reference audio from the upstream - GLM-TTS repository (real human speech matching REF_TEXT). """ synth_text = "今天天气真不错，适合出去散散步。" prompt_audio = _load_ref_audio() @@ -144,5 +136,5 @@ def test_offline_voice_clone_zh(async_chunk: bool) -> None: audio_format="audio/wav", ), {"input": synth_text, "response_format": "wav"}, - run_level="advanced_model", + run_level="full_model", ) diff --git a/tests/e2e/offline_inference/test_higgs_audio_v2.py b/tests/e2e/offline_inference/test_higgs_audio_v2_expansion.py similarity index 96% rename from tests/e2e/offline_inference/test_higgs_audio_v2.py rename to tests/e2e/offline_inference/test_higgs_audio_v2_expansion.py index fdeccc924..00482a132 100644 --- a/tests/e2e/offline_inference/test_higgs_audio_v2.py +++ b/tests/e2e/offline_inference/test_higgs_audio_v2_expansion.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -End-to-end offline inference tests for higgs-audio v2. +E2E Offline expansion tests for higgs-audio v2. Mirrors examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py: build Stage-0 prompt_token_ids with the model's own processor + plain-text @@ -34,6 +34,8 @@ from tests.helpers.media import get_asset_path from tests.helpers.runtime import OmniRunner from tests.helpers.stage_config import get_deploy_config_path +pytestmark = [pytest.mark.slow, pytest.mark.tts] + MODEL = "bosonai/higgs-audio-v2-generation-3B-base" STAGE_CONFIG = get_deploy_config_path("higgs_audio_v2.yaml") SAMPLE_RATE = 24_000 @@ -89,9 +91,7 @@ def _extract_pcm(outputs) -> torch.Tensor: raise AssertionError(f"no audio payload in any stage output (got {len(outputs)} stages)") -@pytest.mark.core_model -@pytest.mark.tts -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_runner_function", [_OMNI_RUNNER_PARAM], indirect=True) def test_higgs_audio_v2_offline_plain_text(omni_runner_function: OmniRunner) -> None: """ @@ -131,9 +131,7 @@ def test_higgs_audio_v2_offline_plain_text(omni_runner_function: OmniRunner) -> assert np.isfinite(arr).all(), "audio contains non-finite samples (nan/inf)" -@pytest.mark.core_model -@pytest.mark.tts -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_runner_function", [_OMNI_RUNNER_PARAM], indirect=True) def test_higgs_audio_v2_offline_batch_two_prompts(omni_runner_function: OmniRunner) -> None: """ @@ -199,9 +197,7 @@ def _compose_voice_clone_request(model_name: str, text: str, ref_text: str, ref_ } -@pytest.mark.advanced_model -@pytest.mark.tts -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_runner_function", [_OMNI_RUNNER_PARAM], indirect=True) def test_higgs_audio_v2_offline_voice_clone(omni_runner_function: OmniRunner) -> None: """ diff --git a/tests/e2e/offline_inference/test_magi_human_expansion.py b/tests/e2e/offline_inference/test_magi_human_expansion.py index 1b3550913..5ce016988 100644 --- a/tests/e2e/offline_inference/test_magi_human_expansion.py +++ b/tests/e2e/offline_inference/test_magi_human_expansion.py @@ -36,7 +36,7 @@ _OMNI_RUNNER_PARAM_OFFLOAD = ( ) pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize( "omni_runner", diff --git a/tests/e2e/offline_inference/test_mammoth_moda2_expansion.py b/tests/e2e/offline_inference/test_mammoth_moda2_expansion.py index 931ce5931..b0d82dbc4 100644 --- a/tests/e2e/offline_inference/test_mammoth_moda2_expansion.py +++ b/tests/e2e/offline_inference/test_mammoth_moda2_expansion.py @@ -99,7 +99,7 @@ def _sample_pixels(img_tensor: torch.Tensor) -> list[float]: # End-to-end test # --------------------------------------------------------------------------- pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/offline_inference/test_ming_flash_omni_expansion.py b/tests/e2e/offline_inference/test_ming_flash_omni_expansion.py index 19cf6fa4c..6d4bd197a 100644 --- a/tests/e2e/offline_inference/test_ming_flash_omni_expansion.py +++ b/tests/e2e/offline_inference/test_ming_flash_omni_expansion.py @@ -49,7 +49,7 @@ test_params_thinker = [(m, c) for m in models for c in [get_eager_config_thinker test_params = [(m, c) for m in models for c in [get_eager_config()]] -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=4) @pytest.mark.parametrize("omni_runner", test_params_thinker, indirect=True) @@ -65,7 +65,7 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=4) @pytest.mark.parametrize("omni_runner", test_params_thinker, indirect=True) @@ -82,7 +82,7 @@ def test_image_to_text(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=4) @pytest.mark.parametrize("omni_runner", test_params_thinker, indirect=True) @@ -101,7 +101,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=4) @pytest.mark.parametrize("omni_runner", test_params_thinker, indirect=True) @@ -118,7 +118,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=4) @pytest.mark.parametrize("omni_runner", test_params_thinker, indirect=True) @@ -138,7 +138,7 @@ def test_mixed_to_text(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=4) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) @@ -154,7 +154,7 @@ def test_text_to_audio(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=4) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) diff --git a/tests/e2e/offline_inference/test_ming_tts.py b/tests/e2e/offline_inference/test_ming_tts_expansion.py similarity index 97% rename from tests/e2e/offline_inference/test_ming_tts.py rename to tests/e2e/offline_inference/test_ming_tts_expansion.py index 775942274..82da74227 100644 --- a/tests/e2e/offline_inference/test_ming_tts.py +++ b/tests/e2e/offline_inference/test_ming_tts_expansion.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""End-to-end offline inference tests for Ming-omni-tts.""" +"""E2E expansion tests for Ming-omni-tts offline inference (nightly CI).""" import asyncio import uuid @@ -32,6 +32,12 @@ TEST_TEXT = "我会一直在这里陪着你，直到你慢慢地沉入那个最 TEST_INSTRUCTION = "轻柔的ASMR耳语，慢速，贴近麦克风" MIN_AUDIO_SAMPLES = 1000 +pytestmark = [ + pytest.mark.slow, + pytest.mark.tts, + pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4704"), +] + @pytest.fixture(scope="module") def ming_tokenizer(): @@ -163,8 +169,6 @@ def _extract_multimodal_output(output) -> Mapping[str, Any]: raise AssertionError("No multimodal audio output found in Ming generate results") -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_ming_tts_offline_basic(ming_engine, ming_tokenizer) -> None: """Test blocking Ming generation through Omni.""" @@ -185,8 +189,6 @@ def test_ming_tts_offline_basic(ming_engine, ming_tokenizer) -> None: assert sample_rate == SAMPLE_RATE, f"Expected Ming output sample rate {SAMPLE_RATE}, got {sample_rate}" -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_ming_tts_speaker_conditioning_differs(ming_engine, ming_tokenizer) -> None: """Test that different Ming speaker controls produce different waveform outputs.""" @@ -220,8 +222,6 @@ def test_ming_tts_speaker_conditioning_differs(ming_engine, ming_tokenizer) -> N ) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_ming_tts_multiple_prompts_queued(ming_engine, ming_tokenizer) -> None: """Regression: supported max_num_seqs=1 config must still drain queued prompts.""" @@ -251,8 +251,6 @@ def test_ming_tts_multiple_prompts_queued(ming_engine, ming_tokenizer) -> None: assert np.max(np.abs(waveform.numpy())) > 0.01, f"Request {i} audio appears silent" -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_ming_tts_offline_streaming(async_omni_engine, ming_tokenizer) -> None: """Test async_chunk streaming Ming generation through AsyncOmni.""" diff --git a/tests/e2e/offline_inference/test_moss_tts.py b/tests/e2e/offline_inference/test_moss_tts_expansion.py similarity index 67% rename from tests/e2e/offline_inference/test_moss_tts.py rename to tests/e2e/offline_inference/test_moss_tts_expansion.py index f8b4794bd..73e919d1c 100644 --- a/tests/e2e/offline_inference/test_moss_tts.py +++ b/tests/e2e/offline_inference/test_moss_tts_expansion.py @@ -2,9 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """E2E offline inference tests for MOSS-VoiceGenerator (MossTTSDelayModel, 1.7B). -Uses the standard omni_runner + pytestmark pattern (one module-scoped engine -per file, skill invariant I4). The realtime variant lives in the sibling file -test_moss_tts_realtime.py. +Weekly expansion coverage (``pytest.mark.slow``) for the delay-model path. +Upstream keeps the same tests in ``test_moss_tts.py``; this branch uses the +``*_expansion.py`` naming convention and deletes the base file on merge. MOSS-VoiceGenerator synthesizes speech from a text *instruction* that describes the desired voice (e.g. "a warm female voice with an American accent"). It does @@ -14,6 +14,8 @@ pair into the (prompt_token_ids, codes.ref) grid that the MossTTSDelayModel talker expects — same as examples/offline_inference/text_to_speech/moss_tts/ end2end.py:_build_unified_codes. +Realtime coverage: ``test_moss_tts_realtime_expansion.py``. + No determinism test: VoiceGenerator produces variable-length output even with a fixed seed; waveform length reproducibility is not guaranteed. """ @@ -31,10 +33,6 @@ from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunner from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - MODEL = "OpenMOSS-Team/MOSS-VoiceGenerator" SAMPLE_RATE = 24_000 @@ -58,18 +56,9 @@ _STAGE1_PARAMS = SamplingParams( ) _SAMPLING = [_STAGE0_PARAMS, _STAGE1_PARAMS] -# --------------------------------------------------------------------------- -# Deploy config -# --------------------------------------------------------------------------- - def _get_test_config() -> str: - """Derive a CI-friendly config from moss_voice_generator.yaml. - - Reduces Stage 0 gpu_memory_utilization from 0.60 → 0.45 to leave headroom - for Stage 1 (0.12) on a shared L4/A10G. max_num_seqs=1 keeps per-test - peak memory predictable. - """ + """Derive a CI-friendly config from moss_voice_generator.yaml.""" return modify_stage_config( get_deploy_config_path("moss_voice_generator.yaml"), updates={ @@ -83,35 +72,16 @@ def _get_test_config() -> str: ) -# --------------------------------------------------------------------------- -# pytestmark — one engine for the whole module -# --------------------------------------------------------------------------- - pytestmark = [ + pytest.mark.slow, pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"), - pytest.mark.full_model, pytest.mark.tts, pytest.mark.parametrize("omni_runner", [(MODEL, _get_test_config())], indirect=True), ] -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - def _build_request(text: str, instruction: str) -> dict: - """Build a VoiceGenerator request using AutoProcessor. - - Calls proc.build_user_message + proc() to produce the unified-codes grid - (L, 1+n_vq) and splits it into prompt_token_ids + codes.ref, exactly as - end2end.py:_build_unified_codes does for the voice_generator variant. - The processor (including its CPU-resident audio_tokenizer) is freed before - returning so it does not compete with the running engine. - - Set MOSS_TTS_SKIP_ON_NET_FAIL=1 to skip in air-gapped environments where - the HF snapshot is not cached. - """ + """Build a VoiceGenerator request using AutoProcessor.""" from transformers import AutoProcessor try: @@ -124,9 +94,9 @@ def _build_request(text: str, instruction: str) -> dict: user_msg = proc.build_user_message(text=text, instruction=instruction) batch = proc(conversations=[[user_msg]], mode="generation") - unified = batch["input_ids"][0] # (L, 1+n_vq) + unified = batch["input_ids"][0] text_ids = unified[:, 0].tolist() - audio_codes = unified[:, 1:].contiguous().to(torch.int64) # (L, n_vq) + audio_codes = unified[:, 1:].contiguous().to(torch.int64) del proc gc.collect() @@ -137,15 +107,6 @@ def _build_request(text: str, instruction: str) -> dict: def _collect_audio(omni_runner: OmniRunner, request: dict) -> tuple[torch.Tensor, int]: - """Run one request and return (waveform_cpu, sample_rate). - - Iterates all OmniRequestOutputs from generate() via the top-level - multimodal_output property (which aggregates from completion outputs). - AR-stage outputs have empty multimodal_output and are silently skipped. - With async_chunk=True and FINAL_ONLY, the codec stage consolidates its - chunks into a single tensor before yielding — but we handle list fallback - for robustness. - """ chunks: list[torch.Tensor] = [] sr_final = SAMPLE_RATE for out in omni_runner.omni.generate(request, _SAMPLING): @@ -172,14 +133,8 @@ def _collect_audio(omni_runner: OmniRunner, request: dict) -> tuple[torch.Tensor return torch.cat(chunks, dim=0), sr_final -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -@pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_moss_tts_english(omni_runner: OmniRunner) -> None: +def test_moss_tts_delay_english(omni_runner: OmniRunner) -> None: """VoiceGenerator: English instruction produces non-empty 24 kHz audio.""" req = _build_request( "Hello, this is a MOSS voice design test.", @@ -192,9 +147,8 @@ def test_moss_tts_english(omni_runner: OmniRunner) -> None: assert not torch.all(audio == 0), "Audio is silence" -@pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_moss_tts_chinese(omni_runner: OmniRunner) -> None: +def test_moss_tts_delay_chinese(omni_runner: OmniRunner) -> None: """VoiceGenerator: Chinese input produces non-empty audio.""" req = _build_request( "你好，这是语音合成测试。", @@ -207,9 +161,8 @@ def test_moss_tts_chinese(omni_runner: OmniRunner) -> None: assert not torch.all(audio == 0) -@pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_moss_tts_batch(omni_runner: OmniRunner) -> None: +def test_moss_tts_delay_batch(omni_runner: OmniRunner) -> None: """VoiceGenerator: batch of two requests each returns non-empty audio.""" requests = [ _build_request("First sentence.", "a warm female voice"), @@ -233,9 +186,6 @@ def test_moss_tts_batch(omni_runner: OmniRunner) -> None: if isinstance(audio, torch.Tensor) and audio.numel() > 0: results.append(audio.reshape(-1).cpu()) - # With async_chunk streaming + FINAL_ONLY, each request produces one - # consolidated output. The >= 2 guard handles backward-compatibility if - # consolidation behavior changes. assert len(results) >= 2, f"Expected at least 2 audio outputs, got {len(results)}" for i, audio in enumerate(results): assert audio.numel() > 0, f"Audio chunk[{i}] is empty" diff --git a/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py b/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py index 2763adf4f..2d247f691 100644 --- a/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py +++ b/tests/e2e/offline_inference/test_moss_tts_nano_expansion.py @@ -32,7 +32,7 @@ _OMNI_RUNNER_PARAM = ( ) pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.tts, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py b/tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py new file mode 100644 index 000000000..80b12e2e4 --- /dev/null +++ b/tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""E2E offline inference tests for MOSS-TTS-Realtime (MossTTSRealtime, 1.7B). + +Exercises the local-transformer generation path on L4. Delay-model coverage is in +``test_moss_tts_expansion.py``. + +No determinism test: MossTTSRealtime is sensitive to async scheduling timing; +waveform reproducibility is not guaranteed across sequential ``generate`` calls. +""" + +from __future__ import annotations + +import os +import urllib.request + +import pytest +import torch +from vllm import SamplingParams + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniRunner +from tests.helpers.stage_config import get_deploy_config_path + +MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime" +DEPLOY_CONFIG = get_deploy_config_path("moss_tts_realtime.yaml") +_OMNI_RUNNER_PARAM = (MODEL, DEPLOY_CONFIG, {"stage_init_timeout": 300}) + +pytestmark = [ + pytest.mark.slow, + pytest.mark.tts, + pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"), + pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), +] + +SAMPLE_RATE = 24_000 +REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav" + +_DEFAULT_SAMPLING = SamplingParams( + temperature=1.7, + top_p=0.8, + top_k=25, + max_tokens=512, + seed=42, + detokenize=False, +) + + +@pytest.fixture(scope="session") +def ref_audio_path(tmp_path_factory) -> str: + cache_dir = tmp_path_factory.mktemp("moss_tts_realtime_ref") + target = cache_dir / "zh_1.wav" + try: + with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp: + target.write_bytes(resp.read()) + except Exception as exc: + msg = f"Cannot fetch reference clip {REF_AUDIO_URL}: {exc}" + if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"): + pytest.skip(msg) + pytest.fail(msg) + if not target.exists() or target.stat().st_size == 0: + pytest.fail(f"Reference clip empty after download: {target}") + return str(target) + + +def _build_request(text: str, ref_audio_path: str, seed: int = 42) -> dict: + return { + "prompt": "<|im_start|>assistant\n", + "additional_information": { + "task_type": ["voice_clone"], + "text": [text], + "mode": ["voice_clone"], + "prompt_audio_path": [ref_audio_path], + "seed": [seed], + }, + } + + +def _sampling_for(omni_runner: OmniRunner) -> SamplingParams | list[SamplingParams]: + omni = omni_runner.omni + if omni.num_stages == 1: + return _DEFAULT_SAMPLING + params = omni_runner.get_default_sampling_params_list() + params[0] = _DEFAULT_SAMPLING + return params + + +def _audio_from_stage(stage_outputs) -> tuple[torch.Tensor, int] | None: + mm = stage_outputs.multimodal_output + if not mm: + return None + audio = mm.get("audio") + if audio is None: + audio = mm.get("model_outputs") + if audio is None: + return None + if isinstance(audio, list): + audio = torch.cat( + [t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0], + dim=0, + ) + if not isinstance(audio, torch.Tensor) or audio.numel() == 0: + return None + sr = mm.get("sr") + sample_rate = int(sr.item()) if sr is not None else SAMPLE_RATE + return audio.reshape(-1).cpu(), sample_rate + + +def _collect_audio(omni_runner: OmniRunner, request: dict) -> tuple[torch.Tensor, int]: + for stage_outputs in omni_runner.omni.generate(request, _sampling_for(omni_runner)): + parsed = _audio_from_stage(stage_outputs) + if parsed is not None: + return parsed + raise AssertionError("No stage outputs received") + + +@hardware_test(res={"cuda": "L4"}) +def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path) -> None: + """MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio.""" + req = _build_request("This is a real-time TTS streaming test.", ref_audio_path) + audio, sr = _collect_audio(omni_runner, req) + + assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}" + assert audio.numel() > 0, "Audio tensor is empty" + assert not torch.all(audio == 0), "Audio is silence" diff --git a/tests/e2e/offline_inference/test_moss_tts_v1_5.py b/tests/e2e/offline_inference/test_moss_tts_v1_5.py deleted file mode 100644 index 99c3b7760..000000000 --- a/tests/e2e/offline_inference/test_moss_tts_v1_5.py +++ /dev/null @@ -1,113 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""E2E offline inference test for MOSS-TTS-v1.5 (MossTTSDelay-8B). - -MOSS-TTS-v1.5 is a continued-training upgrade of MOSS-TTS 1.0 with the SAME -``MossTTSDelay`` architecture and API, so it reuses the existing talker/codec -path unchanged — this test just pins that the 8B v1.5 checkpoint loads and -produces audio. - -The model is 8B, so this lives in its own file (one module-scoped engine per -file, per skill invariant I4) and is gated behind an 80 GB GPU; the small-GPU -MossTTSDelay coverage stays in ``test_moss_tts.py`` (1.7B VoiceGenerator). -""" - -from __future__ import annotations - -import os -import urllib.request - -import pytest -import torch -from vllm import SamplingParams - -from tests.helpers.mark import hardware_test -from vllm_omni import Omni - -# H100-gated (8B). Picked up by the nightly "TTS · Function Test with H100" -# job (-m "full_model and H100 and tts"); the L4 MOSS-TTS tests land in the -# sibling L4 tts lane instead. -# TODO: Fix this test -# pytestmark = [pytest.mark.full_model, pytest.mark.tts] - -SAMPLE_RATE = 24_000 -REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav" -_MODEL = "OpenMOSS-Team/MOSS-TTS-v1.5" - -_DEFAULT_SAMPLING = SamplingParams( - temperature=1.7, - top_p=0.8, - top_k=25, - max_tokens=512, # short for tests (~20 s at 24 kHz, ~12.5 tokens/s) - seed=42, - detokenize=False, -) - - -@pytest.fixture(scope="session") -def ref_audio_path(tmp_path_factory) -> str: - """Download the upstream reference clip once per session. - - Set ``MOSS_TTS_SKIP_ON_NET_FAIL=1`` to skip in air-gapped environments. - """ - target = tmp_path_factory.mktemp("moss_tts_v15_ref") / "zh_1.wav" - try: - with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp: - target.write_bytes(resp.read()) - except Exception as exc: # noqa: BLE001 - msg = f"Cannot fetch reference clip {REF_AUDIO_URL}: {exc}" - if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"): - pytest.skip(msg) - pytest.fail(msg) - if not target.exists() or target.stat().st_size == 0: - pytest.fail(f"Reference clip empty after download: {target}") - return str(target) - - -@pytest.fixture(scope="module") -def v15_engine(): - return Omni(model=_MODEL, stage_init_timeout=600) - - -def _build_request(text: str, ref_audio_path: str) -> dict: - return { - "prompt": "<|im_start|>assistant\n", - "additional_information": { - "task_type": ["voice_clone"], - "text": [text], - "mode": ["voice_clone"], - "prompt_audio_path": [ref_audio_path], - "seed": [42], - }, - } - - -def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]: - for stage_outputs in omni.generate(request, _DEFAULT_SAMPLING): - for req_output in stage_outputs.request_output: - mm = req_output.outputs[0].multimodal_output - assert mm is not None, "Expected non-None multimodal_output" - audio = mm.get("audio") - if audio is None: - audio = mm.get("model_outputs") - sr = mm.get("sr") - assert audio is not None, "Expected 'audio' or 'model_outputs' in multimodal_output" - if isinstance(audio, list): - audio = torch.cat( - [t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0], - dim=0, - ) - assert isinstance(audio, torch.Tensor), f"Expected Tensor, got {type(audio)}" - return audio.reshape(-1).cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE - raise AssertionError("No stage outputs received") - - -@hardware_test(res={"cuda": "H100"}) -def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path): - """MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio.""" - req = _build_request("Hello, this is a MOSS-TTS v1.5 voice cloning test.", ref_audio_path) - audio, sr = _collect_audio(v15_engine, req) - - assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}" - assert audio.numel() > 0, "Audio tensor is empty" - assert not torch.all(audio == 0), "Audio is silence" diff --git a/tests/e2e/offline_inference/test_moss_tts_v1_5_expansion.py b/tests/e2e/offline_inference/test_moss_tts_v1_5_expansion.py new file mode 100644 index 000000000..43bf80415 --- /dev/null +++ b/tests/e2e/offline_inference/test_moss_tts_v1_5_expansion.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""E2E offline inference test for MOSS-TTS-v1.5 (MossTTSDelay-8B). + +MOSS-TTS-v1.5 is a continued-training upgrade of MOSS-TTS 1.0 with the SAME +``MossTTSDelay`` architecture and API. The 8B checkpoint is H100-gated; small-GPU +MossTTSDelay coverage stays in ``test_moss_tts_expansion.py``. +""" + +from __future__ import annotations + +import os +import urllib.request + +import pytest +import torch +from vllm import SamplingParams + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniRunner +from tests.helpers.stage_config import get_deploy_config_path + +MODEL = "OpenMOSS-Team/MOSS-TTS-v1.5" +DEPLOY_CONFIG = get_deploy_config_path("moss_tts.yaml") +_OMNI_RUNNER_PARAM = (MODEL, DEPLOY_CONFIG, {"stage_init_timeout": 600}) + +pytestmark = [ + pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4643"), + pytest.mark.slow, + pytest.mark.tts, + pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), +] + +SAMPLE_RATE = 24_000 +REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav" + +_DEFAULT_SAMPLING = SamplingParams( + temperature=1.7, + top_p=0.8, + top_k=25, + max_tokens=512, + seed=42, + detokenize=False, +) + + +@pytest.fixture(scope="session") +def ref_audio_path(tmp_path_factory) -> str: + target = tmp_path_factory.mktemp("moss_tts_v15_ref") / "zh_1.wav" + try: + with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp: + target.write_bytes(resp.read()) + except Exception as exc: # noqa: BLE001 + msg = f"Cannot fetch reference clip {REF_AUDIO_URL}: {exc}" + if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"): + pytest.skip(msg) + pytest.fail(msg) + if not target.exists() or target.stat().st_size == 0: + pytest.fail(f"Reference clip empty after download: {target}") + return str(target) + + +def _build_request(text: str, ref_audio_path: str) -> dict: + return { + "prompt": "<|im_start|>assistant\n", + "additional_information": { + "task_type": ["voice_clone"], + "text": [text], + "mode": ["voice_clone"], + "prompt_audio_path": [ref_audio_path], + "seed": [42], + }, + } + + +def _sampling_for(omni_runner: OmniRunner) -> SamplingParams | list[SamplingParams]: + omni = omni_runner.omni + if omni.num_stages == 1: + return _DEFAULT_SAMPLING + params = omni_runner.get_default_sampling_params_list() + params[0] = _DEFAULT_SAMPLING + return params + + +def _collect_audio(omni_runner: OmniRunner, request: dict) -> tuple[torch.Tensor, int]: + for stage_outputs in omni_runner.omni.generate(request, _sampling_for(omni_runner)): + mm = stage_outputs.multimodal_output + if not mm: + continue + audio = mm.get("audio") + if audio is None: + audio = mm.get("model_outputs") + if audio is None: + continue + if isinstance(audio, list): + audio = torch.cat( + [t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0], + dim=0, + ) + if not isinstance(audio, torch.Tensor) or audio.numel() == 0: + continue + sr = mm.get("sr") + return audio.reshape(-1).cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE + raise AssertionError("No stage outputs received") + + +@hardware_test(res={"cuda": "H100"}) +def test_moss_tts_v15_voice_clone(omni_runner: OmniRunner, ref_audio_path) -> None: + """MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio.""" + req = _build_request("Hello, this is a MOSS-TTS v1.5 voice cloning test.", ref_audio_path) + audio, sr = _collect_audio(omni_runner, req) + + assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}" + assert audio.numel() > 0, "Audio tensor is empty" + assert not torch.all(audio == 0), "Audio is silence" diff --git a/tests/e2e/offline_inference/test_omnivoice_expansion.py b/tests/e2e/offline_inference/test_omnivoice_expansion.py index 7ea646a85..692278ef7 100644 --- a/tests/e2e/offline_inference/test_omnivoice_expansion.py +++ b/tests/e2e/offline_inference/test_omnivoice_expansion.py @@ -31,7 +31,7 @@ _OMNI_RUNNER_PARAM = ( ) pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.tts, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/offline_inference/test_qwen2_5_omni_autoround_w4a16_expansion.py b/tests/e2e/offline_inference/test_qwen2_5_omni_autoround_w4a16_expansion.py index 190857e3b..6be54969c 100644 --- a/tests/e2e/offline_inference/test_qwen2_5_omni_autoround_w4a16_expansion.py +++ b/tests/e2e/offline_inference/test_qwen2_5_omni_autoround_w4a16_expansion.py @@ -23,7 +23,7 @@ from tests.helpers.media import ( from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.omni, ] @@ -36,6 +36,10 @@ BASELINE_MODEL = os.environ.get("QWEN2_5_OMNI_BASELINE_MODEL", BASELINE_MODEL) _CI_DEPLOY = get_deploy_config_path("ci/qwen2_5_omni.yaml") +_SKIP_ISSUE_4663 = pytest.mark.skip( + reason="https://github.com/vllm-project/vllm-omni/issues/4663", +) + def _get_stage_config(): """Build a CI-friendly stage config with eager mode.""" @@ -61,6 +65,7 @@ quant_params = [(QUANTIZED_MODEL, stage_config)] # ------------------------------------------------------------------ +@_SKIP_ISSUE_4663 @hardware_test(res={"cuda": "L4"}, num_cards=4) @pytest.mark.parametrize("omni_runner", quant_params, indirect=True) def test_text_to_text(omni_runner, omni_runner_handler): @@ -79,6 +84,7 @@ def test_text_to_text(omni_runner, omni_runner_handler): # ------------------------------------------------------------------ +@_SKIP_ISSUE_4663 @hardware_test(res={"cuda": "L4"}, num_cards=4) @pytest.mark.parametrize("omni_runner", quant_params, indirect=True) def test_audio_to_text(omni_runner, omni_runner_handler): @@ -102,6 +108,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler): # ------------------------------------------------------------------ +@_SKIP_ISSUE_4663 @hardware_test(res={"cuda": "L4"}, num_cards=4) @pytest.mark.parametrize("omni_runner", quant_params, indirect=True) def test_image_to_text(omni_runner, omni_runner_handler): @@ -123,6 +130,7 @@ def test_image_to_text(omni_runner, omni_runner_handler): # ------------------------------------------------------------------ +@_SKIP_ISSUE_4663 @hardware_test(res={"cuda": "L4"}, num_cards=4) @pytest.mark.parametrize("omni_runner", quant_params, indirect=True) def test_video_to_text(omni_runner, omni_runner_handler): @@ -144,6 +152,7 @@ def test_video_to_text(omni_runner, omni_runner_handler): # ------------------------------------------------------------------ +@_SKIP_ISSUE_4663 @hardware_test(res={"cuda": "L4"}, num_cards=4) @pytest.mark.parametrize("omni_runner", quant_params, indirect=True) def test_mix_to_audio(omni_runner, omni_runner_handler): diff --git a/tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py b/tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py index 34a878456..9d368a688 100644 --- a/tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py +++ b/tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py @@ -51,7 +51,7 @@ def get_question(prompt_type="mix"): return prompts.get(prompt_type, prompts["mix"]) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 4, "rocm": 2, "xpu": 3}) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) @@ -82,7 +82,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 4, "rocm": 2, "xpu": 3}) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) diff --git a/tests/e2e/offline_inference/test_sensenova_u1_expansion.py b/tests/e2e/offline_inference/test_sensenova_u1_expansion.py index 1f9769f81..4f5c97d11 100644 --- a/tests/e2e/offline_inference/test_sensenova_u1_expansion.py +++ b/tests/e2e/offline_inference/test_sensenova_u1_expansion.py @@ -124,7 +124,7 @@ def _make_file_lora_request(adapter_dir: Path) -> LoRARequest: # --------------------------------------------------------------------------- -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}) def test_sensenova_u1_lora_scale_and_deactivation(tmp_path) -> None: diff --git a/tests/e2e/offline_inference/test_sensenova_u1_img2img.py b/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py similarity index 96% rename from tests/e2e/offline_inference/test_sensenova_u1_img2img.py rename to tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py index fedd8e90e..242d6a12e 100644 --- a/tests/e2e/offline_inference/test_sensenova_u1_img2img.py +++ b/tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py @@ -153,9 +153,9 @@ def _generate_sensenova_u1_img2img( return generated_image -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion +@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4636") @hardware_test(res={"cuda": "H100"}) def test_sensenova_u1_img2img(run_level): """Test SenseNova-U1 img2img (single-stage diffusion, no deploy YAML).""" @@ -165,11 +165,11 @@ def test_sensenova_u1_img2img(run_level): stage_configs_path=None, ) as runner: generated_image = _generate_sensenova_u1_img2img(runner.omni, input_image) - if run_level == "advanced_model": + if run_level == "full_model" or run_level == "advanced_model": _validate_pixels(generated_image) -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.cache @hardware_test(res={"cuda": "H100"}) diff --git a/tests/e2e/offline_inference/test_sensenova_u1_text2img.py b/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py similarity index 96% rename from tests/e2e/offline_inference/test_sensenova_u1_text2img.py rename to tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py index fc9b5411e..65b11d3ee 100644 --- a/tests/e2e/offline_inference/test_sensenova_u1_text2img.py +++ b/tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py @@ -143,9 +143,9 @@ def _generate_sensenova_u1_image( return generated_image -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion +@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4636") @hardware_test(res={"cuda": "H100"}) def test_sensenova_u1_text2img(run_level): """Test SenseNova-U1 text2img (single-stage diffusion, no deploy YAML).""" @@ -154,11 +154,11 @@ def test_sensenova_u1_text2img(run_level): stage_configs_path=None, ) as runner: generated_image = _generate_sensenova_u1_image(runner.omni) - if run_level == "advanced_model": + if run_level == "full_model" or run_level == "advanced_model": _validate_pixels(generated_image) -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.cache @hardware_test(res={"cuda": "H100"}) diff --git a/tests/e2e/offline_inference/test_stable_audio_expansion.py b/tests/e2e/offline_inference/test_stable_audio_expansion.py index 2785dab24..fc3b5596d 100644 --- a/tests/e2e/offline_inference/test_stable_audio_expansion.py +++ b/tests/e2e/offline_inference/test_stable_audio_expansion.py @@ -23,7 +23,7 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.platforms import current_omni_platform -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.slow, pytest.mark.diffusion] _MODEL_REPO = "stabilityai/stable-audio-open-1.0" diff --git a/tests/e2e/offline_inference/test_step_audio2.py b/tests/e2e/offline_inference/test_step_audio2.py deleted file mode 100644 index cd779a6de..000000000 --- a/tests/e2e/offline_inference/test_step_audio2.py +++ /dev/null @@ -1,303 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -E2E tests for Step-Audio2 model with audio input and audio output. - -These tests verify: -1. Processor loads correctly and handles audio input -2. Field configuration for vLLM batching (flat_from_sizes) -3. Tensor shapes are correct (3D not 4D for profiling) -4. Basic E2E inference works with audio input -""" - -from pathlib import Path - -import numpy as np -import pytest -import torch - -from tests.helpers.runtime import OmniRunner - -models = ["stepfun-ai/Step-Audio-2-mini"] - -stage_configs = [str(Path(__file__).parent / "stage_configs" / "step_audio2_ci.yaml")] - -test_params = [(model, stage_config) for model in models for stage_config in stage_configs] - - -def create_dummy_audio(sample_rate: int = 16000, duration_sec: float = 1.0) -> tuple[np.ndarray, int]: - """Create a dummy audio signal (sine wave) for testing.""" - t = np.linspace(0, duration_sec, int(sample_rate * duration_sec), dtype=np.float32) - audio = 0.5 * np.sin(2 * np.pi * 440 * t) - return audio, sample_rate - - -class StepAudio2OmniRunner(OmniRunner): - """Test runner specialized for Step-Audio2 model.""" - - def get_omni_inputs( - self, - prompts: list[str] | str, - system_prompt: str | None = None, - audios=None, - **kwargs, - ) -> list[dict]: - """Construct Omni input format for Step-Audio2.""" - if system_prompt is None: - system_prompt = "你是一个语音对话助手，能够理解音频输入并生成语音回复。" - - audio_placeholder = "" - - if isinstance(prompts, str): - prompts = [prompts] - - num_prompts = len(prompts) - - if audios is None: - audios_list = [None] * num_prompts - elif isinstance(audios, list): - audios_list = audios if len(audios) == num_prompts else [audios[0]] * num_prompts - else: - audios_list = [audios] * num_prompts - - omni_inputs = [] - for i, prompt_text in enumerate(prompts): - user_content = "" - multi_modal_data = {} - - audio = audios_list[i] - if audio is not None: - user_content += audio_placeholder - multi_modal_data["audio"] = audio - - user_content += prompt_text - - full_prompt = ( - f"<|im_start|>system\n{system_prompt}<|im_end|>\n" - f"<|im_start|>user\n{user_content}<|im_end|>\n" - f"<|im_start|>assistant\n" - ) - - input_dict = {"prompt": full_prompt} - if multi_modal_data: - input_dict["multi_modal_data"] = multi_modal_data - - omni_inputs.append(input_dict) - - return omni_inputs - - -@pytest.fixture(scope="session") -def step_audio2_runner(): - return StepAudio2OmniRunner - - -@pytest.mark.core_model -def test_processor_loads_correctly(): - """Test that Step-Audio2 processor can be imported and initialized.""" - from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( - StepAudio2MultiModalProcessor, - StepAudio2ProcessingInfo, - StepAudio2Processor, - ) - - assert StepAudio2MultiModalProcessor is not None - assert StepAudio2ProcessingInfo is not None - assert StepAudio2Processor is not None - - -@pytest.mark.core_model -def test_audio_preprocessing(): - """Test that audio preprocessing produces correct tensor shapes.""" - from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( - log_mel_spectrogram, - padding_mels, - ) - - audio, sample_rate = create_dummy_audio(sample_rate=16000, duration_sec=1.0) - - mel = log_mel_spectrogram(audio) - - assert mel.ndim == 2, f"Expected 2D tensor, got {mel.ndim}D" - assert mel.shape[0] == 128, f"Expected 128 mel bins, got {mel.shape[0]}" - - mels = [mel, mel] - padded_mels, mel_lens = padding_mels(mels) - - assert padded_mels.ndim == 3, f"Expected 3D tensor, got {padded_mels.ndim}D" - assert padded_mels.shape[0] == 2, f"Expected batch size 2, got {padded_mels.shape[0]}" - assert padded_mels.shape[1] == 128, f"Expected 128 mel bins, got {padded_mels.shape[1]}" - - -@pytest.mark.core_model -def test_feature_length_calculation(): - """Test that audio feature length calculation is correct.""" - from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( - calculate_audio_feature_length, - ) - - assert calculate_audio_feature_length(1000) == 125 - - assert calculate_audio_feature_length(100) > 0 - - assert calculate_audio_feature_length(1) >= 1 - - -@pytest.mark.core_model -def test_token_config_constants(): - """Test that token configuration constants are set correctly.""" - from vllm_omni.model_executor.models.step_audio2.step_audio2_constants import ( - DEFAULT_TOKEN_CONFIG, - STEP_AUDIO2_AUDIO_END, - STEP_AUDIO2_AUDIO_PATCH_TOKEN_ID, - STEP_AUDIO2_AUDIO_START, - STEP_AUDIO2_AUDIO_VOCAB_SIZE, - STEP_AUDIO2_TEXT_MAX, - ) - - assert STEP_AUDIO2_TEXT_MAX < STEP_AUDIO2_AUDIO_START, "Text tokens should come before audio tokens" - assert STEP_AUDIO2_AUDIO_END == STEP_AUDIO2_AUDIO_START + STEP_AUDIO2_AUDIO_VOCAB_SIZE - 1 - assert STEP_AUDIO2_AUDIO_PATCH_TOKEN_ID > STEP_AUDIO2_TEXT_MAX - assert STEP_AUDIO2_AUDIO_PATCH_TOKEN_ID < STEP_AUDIO2_AUDIO_START - - assert DEFAULT_TOKEN_CONFIG.text_max == STEP_AUDIO2_TEXT_MAX - assert DEFAULT_TOKEN_CONFIG.audio_start == STEP_AUDIO2_AUDIO_START - - -@pytest.mark.core_model -def test_mm_field_config_structure(): - """Test that multimodal field config is properly structured for vLLM batching.""" - from vllm.multimodal.inputs import MultiModalFieldConfig - - audio_lens = torch.tensor([100, 200, 150], dtype=torch.int32) - - field_config = MultiModalFieldConfig.flat_from_sizes("audio", audio_lens) - - assert field_config is not None - - -@pytest.mark.core_model -def test_audio_encoder_output_shape(): - """Test that audio encoder produces correct output shapes.""" - from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( - Adaptor, - AudioEncoder, - ) - - encoder = AudioEncoder(n_mels=128, n_ctx=1500, n_state=512, n_head=8, n_layer=6) - - batch_size = 2 - n_mels = 128 - time_steps = 400 - x = torch.randn(batch_size, n_mels, time_steps) - x_len = torch.tensor([time_steps, time_steps // 2], dtype=torch.int32) - - encoded, encoded_lens = encoder(x, x_len) - - assert encoded.ndim == 3, f"Expected 3D tensor, got {encoded.ndim}D" - assert encoded.shape[0] == batch_size - assert encoded.shape[2] == 512 - - adapter = Adaptor(n_state=512, n_hidden=4096, kernel_size=3, stride=2) - adapted = adapter(encoded) - - assert adapted.ndim == 3 - assert adapted.shape[0] == batch_size - assert adapted.shape[2] == 4096 - - -@pytest.mark.core_model -def test_token_separation(): - """Test that token separation works correctly.""" - from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( - StepAudio2ThinkerForConditionalGeneration, - ) - - token_ids = [100, 200, 151700, 300, 151800, 400] - - text_tokens, audio_tokens = StepAudio2ThinkerForConditionalGeneration.separate_tokens(token_ids) - - assert text_tokens == [100, 200, 300, 400] - assert audio_tokens == [4, 104] - - -@pytest.mark.core_model -def test_has_audio_output(): - """Test detection of audio tokens in output.""" - from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( - StepAudio2ThinkerForConditionalGeneration, - ) - - text_only = [100, 200, 300] - assert not StepAudio2ThinkerForConditionalGeneration.has_audio_output(text_only) - - with_audio = [100, 200, 151700, 300] - assert StepAudio2ThinkerForConditionalGeneration.has_audio_output(with_audio) - - -@pytest.mark.advanced_model -@pytest.mark.parametrize("test_config", test_params) -def test_audio_to_text_and_audio(step_audio2_runner: type[StepAudio2OmniRunner], test_config: tuple[str, str]) -> None: - """Test processing audio input and generating text + audio output.""" - model, stage_config_path = test_config - - with step_audio2_runner(model, seed=42, stage_configs_path=stage_config_path) as runner: - audio = create_dummy_audio(sample_rate=16000, duration_sec=2.0) - - outputs = runner.generate_multimodal( - prompts="请复述这段音频的内容。", - audios=audio, - ) - - assert len(outputs) > 0 - - text_output = None - for stage_output in outputs: - if stage_output.final_output_type == "text": - text_output = stage_output - break - - assert text_output is not None - assert text_output.request_output is not None - text_content = text_output.request_output.outputs[0].text - assert text_content is not None - assert len(text_content.strip()) > 0 - - audio_output = None - for stage_output in outputs: - if stage_output.final_output_type == "audio": - audio_output = stage_output - break - - if audio_output is not None: - assert audio_output.request_output is not None - audio_tensor = audio_output.request_output.outputs[0].multimodal_output.get("audio") - if audio_tensor is not None: - assert audio_tensor.numel() > 0 - - -@pytest.mark.advanced_model -@pytest.mark.parametrize("test_config", test_params) -def test_text_only_input(step_audio2_runner: type[StepAudio2OmniRunner], test_config: tuple[str, str]) -> None: - """Test processing text-only input (no audio).""" - model, stage_config_path = test_config - - with step_audio2_runner(model, seed=42, stage_configs_path=stage_config_path) as runner: - outputs = runner.generate_multimodal( - prompts="你好，请用中文回答。", - audios=None, - ) - - assert len(outputs) > 0 - - text_output = None - for stage_output in outputs: - if stage_output.final_output_type == "text": - text_output = stage_output - break - - assert text_output is not None - assert text_output.request_output is not None - text_content = text_output.request_output.outputs[0].text - assert text_content is not None - assert len(text_content.strip()) > 0 diff --git a/tests/e2e/offline_inference/test_step_audio2_expansion.py b/tests/e2e/offline_inference/test_step_audio2_expansion.py new file mode 100644 index 000000000..362f2f33f --- /dev/null +++ b/tests/e2e/offline_inference/test_step_audio2_expansion.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E expansion tests for Step-Audio2 offline inference (nightly CI). + +Full-model inference with audio input and audio output. +""" + +from pathlib import Path +from typing import Any + +import numpy as np +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_audio +from tests.helpers.runtime import OmniRunner +from vllm_omni.outputs import OmniRequestOutput + +MODEL = "stepfun-ai/Step-Audio-2-mini" +STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "step_audio2_ci.yaml") +TEST_PARAMS = [(MODEL, STAGE_CONFIG)] + +pytestmark = [pytest.mark.slow, pytest.mark.tts] + +DEFAULT_SYSTEM_PROMPT = "你是一个语音对话助手，能够理解音频输入并生成语音回复。" +SAMPLE_RATE = 16000 + + +def _synthetic_audio(duration_sec: int = 2) -> tuple[np.ndarray, int]: + audio = generate_synthetic_audio(duration_sec, 1, SAMPLE_RATE)["np_array"] + if len(audio.shape) == 2: + audio = audio.squeeze() + return np.asarray(audio, dtype=np.float32), SAMPLE_RATE + + +def _build_step_audio2_input( + user_text: str, + *, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, + audio: tuple[np.ndarray, int] | None = None, +) -> dict[str, Any]: + user_content = "" + multi_modal_data: dict[str, Any] = {} + if audio is not None: + user_content += "" + multi_modal_data["audio"] = audio + user_content += user_text + + full_prompt = ( + f"<|im_start|>system\n{system_prompt}<|im_end|>\n" + f"<|im_start|>user\n{user_content}<|im_end|>\n" + f"<|im_start|>assistant\n" + ) + input_dict: dict[str, Any] = {"prompt": full_prompt} + if multi_modal_data: + input_dict["multi_modal_data"] = multi_modal_data + return input_dict + + +def _assert_non_empty_text_output(outputs: list[OmniRequestOutput]) -> None: + assert len(outputs) > 0 + + text_output = next((o for o in outputs if o.final_output_type == "text"), None) + assert text_output is not None + assert text_output.request_output is not None + text_content = text_output.request_output.outputs[0].text + assert text_content is not None + assert len(text_content.strip()) > 0 + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_runner", TEST_PARAMS, indirect=True) +def test_audio_to_text_and_audio(omni_runner: OmniRunner) -> None: + """Test processing audio input and generating text + audio output.""" + audio = _synthetic_audio(duration_sec=2) + omni_input = _build_step_audio2_input("请复述这段音频的内容。", audio=audio) + outputs = omni_runner.generate([omni_input]) + + _assert_non_empty_text_output(outputs) + + audio_output = next((o for o in outputs if o.final_output_type == "audio"), None) + if audio_output is not None: + assert audio_output.request_output is not None + audio_tensor = audio_output.request_output.outputs[0].multimodal_output.get("audio") + if audio_tensor is not None: + assert audio_tensor.numel() > 0 + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_runner", TEST_PARAMS, indirect=True) +def test_text_only_input(omni_runner: OmniRunner) -> None: + """Test processing text-only input (no audio).""" + omni_input = _build_step_audio2_input("你好，请用中文回答。") + outputs = omni_runner.generate([omni_input]) + _assert_non_empty_text_output(outputs) diff --git a/tests/e2e/offline_inference/test_voxcpm2_tts.py b/tests/e2e/offline_inference/test_voxcpm2_tts.py index 817be4064..5c91db45a 100644 --- a/tests/e2e/offline_inference/test_voxcpm2_tts.py +++ b/tests/e2e/offline_inference/test_voxcpm2_tts.py @@ -14,12 +14,10 @@ VOXCPM2_MODEL = "openbmb/VoxCPM2" DEPLOY_CONFIG = get_deploy_config_path("voxcpm2.yaml") SAMPLE_RATE = 48000 +# (model, stage_config_path) — see ``omni_runner`` in tests.helpers.fixtures.runtime +_OMNI_RUNNER_PARAM = (VOXCPM2_MODEL, DEPLOY_CONFIG) -@pytest.fixture(scope="module") -def voxcpm2_engine(): - """Create VoxCPM2 engine for testing.""" - with OmniRunner(VOXCPM2_MODEL, stage_configs_path=DEPLOY_CONFIG) as runner: - yield runner.omni +pytestmark = pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True) def _extract_audio(multimodal_output: dict) -> torch.Tensor: @@ -45,9 +43,9 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor: @pytest.mark.advanced_model @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_voxcpm2_zero_shot_001(voxcpm2_engine): +def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None: """Test zero-shot TTS produces valid audio output.""" - outputs = voxcpm2_engine.generate([{"prompt": "Hello, this is a test."}]) + outputs = omni_runner.omni.generate([{"prompt": "Hello, this is a test."}]) assert len(outputs) == 1 audio = _extract_audio(outputs[0].outputs[0].multimodal_output) @@ -59,7 +57,7 @@ def test_voxcpm2_zero_shot_001(voxcpm2_engine): @pytest.mark.advanced_model @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_voxcpm2_voice_clone_002(voxcpm2_engine): +def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None: """Test voice cloning with a reference audio file. Uses the example ``reference_speaker.wav`` bundled with the voxcpm @@ -82,7 +80,7 @@ def test_voxcpm2_voice_clone_002(voxcpm2_engine): if ref_path is None: pytest.skip("No reference audio available for voice clone test") - outputs = voxcpm2_engine.generate( + outputs = omni_runner.omni.generate( [ { "prompt": "Hello, this is a voice clone demo.", @@ -101,7 +99,7 @@ def test_voxcpm2_voice_clone_002(voxcpm2_engine): @pytest.mark.advanced_model @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_voxcpm2_prefill_decode_mixed_batch_003(voxcpm2_engine): +def test_voxcpm2_prefill_decode_mixed_batch_003(omni_runner: OmniRunner) -> None: """Regression: prefill+decode mixed batch must not crash (PR #2903).""" long_prompt = ( "This is a deliberately long prompt that will stay in the decode " @@ -117,7 +115,7 @@ def test_voxcpm2_prefill_decode_mixed_batch_003(voxcpm2_engine): ] requests = [{"prompt": long_prompt}] + [{"prompt": p} for p in short_prompts] - outputs = voxcpm2_engine.generate(requests) + outputs = omni_runner.omni.generate(requests) assert len(outputs) == len(requests) for i, out in enumerate(outputs): diff --git a/tests/e2e/offline_inference/test_voxtral_tts.py b/tests/e2e/offline_inference/test_voxtral_tts_expansion.py similarity index 82% rename from tests/e2e/offline_inference/test_voxtral_tts.py rename to tests/e2e/offline_inference/test_voxtral_tts_expansion.py index 53ac80b78..56529dbbe 100644 --- a/tests/e2e/offline_inference/test_voxtral_tts.py +++ b/tests/e2e/offline_inference/test_voxtral_tts_expansion.py @@ -2,16 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -End-to-end offline inference tests for Voxtral TTS model. +E2E expansion tests for Voxtral TTS offline inference (nightly CI). Tests both synchronous (Omni) and async streaming (AsyncOmni) paths. - -Equivalent to running: - python3 examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \ - --model mistralai/Voxtral-4B-TTS-2603 \ - --voice casual_female \ - --text "Hello, how are you?" \ - --write-audio """ import asyncio @@ -36,14 +29,12 @@ from vllm_omni.entrypoints.async_omni import AsyncOmni MODEL = "mistralai/Voxtral-4B-TTS-2603" STAGE_CONFIG = get_deploy_config_path("voxtral_tts.yaml") -SAMPLE_RATE = 24000 -# Minimum expected audio samples for a short sentence (~0.04s of 24kHz audio) MIN_AUDIO_SAMPLES = 1000 VOICE = "casual_female" TEST_TEXT = "Hello, how are you?" -# (model, stage_config_path, extra_omni_kwargs) for indirect parametrize on -# ``omni_runner_function`` (function-scoped: must exit before ``AsyncOmni`` test). +pytestmark = [pytest.mark.slow, pytest.mark.tts] + _OMNI_RUNNER_PARAM = ( MODEL, STAGE_CONFIG, @@ -66,10 +57,8 @@ def _compose_request(model_name: str, text: str, voice: str) -> dict: } -@pytest.mark.advanced_model -@pytest.mark.tts @pytest.mark.parametrize("omni_runner_function", [_OMNI_RUNNER_PARAM], indirect=True) -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voxtral_tts_offline_basic(omni_runner_function: OmniRunner) -> None: """Offline sync path; function-scoped runner so AsyncOmni test does not overlap a live Omni.""" omni = omni_runner_function.omni @@ -82,7 +71,6 @@ def test_voxtral_tts_offline_basic(omni_runner_function: OmniRunner) -> None: assert len(outputs) > 0, "No outputs generated" - # Find audio output from the final stage audio_data = None for o in outputs: mm = getattr(o, "multimodal_output", None) @@ -92,7 +80,6 @@ def test_voxtral_tts_offline_basic(omni_runner_function: OmniRunner) -> None: assert audio_data is not None, "No audio output found in any stage output" - # Concatenate audio chunks if returned as a list of tensors if isinstance(audio_data, list): audio_tensor = torch.cat(audio_data) elif isinstance(audio_data, torch.Tensor): @@ -105,14 +92,10 @@ def test_voxtral_tts_offline_basic(omni_runner_function: OmniRunner) -> None: assert len(audio_array) > MIN_AUDIO_SAMPLES, ( f"Audio too short: {len(audio_array)} samples, expected > {MIN_AUDIO_SAMPLES}" ) - - # Verify audio isn't all zeros / silence assert np.max(np.abs(audio_array)) > 0.01, "Audio appears to be silence" -@pytest.mark.advanced_model -@pytest.mark.tts -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voxtral_tts_offline_streaming(): """Test AsyncOmni streaming inference for Voxtral TTS.""" @@ -126,7 +109,6 @@ def test_voxtral_tts_offline_streaming(): try: inputs = _compose_request(MODEL, TEST_TEXT, VOICE) - sampling_params = SamplingParams(max_tokens=2500) all_audio_chunks = [] @@ -146,8 +128,6 @@ def test_voxtral_tts_offline_streaming(): if isinstance(audio_chunk, torch.Tensor): if finished: - # Last chunk may return whole audio instead of - # a delta — cut already-accumulated samples. audio_np = audio_chunk[accumulated_samples:].float().detach().cpu().numpy() else: audio_np = audio_chunk.float().detach().cpu().numpy() @@ -167,8 +147,6 @@ def test_voxtral_tts_offline_streaming(): assert len(audio_array) > MIN_AUDIO_SAMPLES, ( f"Audio too short: {len(audio_array)} samples, expected > {MIN_AUDIO_SAMPLES}" ) - - # Verify audio isn't all zeros / silence assert np.max(np.abs(audio_array)) > 0.01, "Audio appears to be silence" finally: diff --git a/tests/e2e/online_serving/test_audiox_expansion.py b/tests/e2e/online_serving/test_audiox_expansion.py index ae809d13e..fc5bf06cf 100644 --- a/tests/e2e/online_serving/test_audiox_expansion.py +++ b/tests/e2e/online_serving/test_audiox_expansion.py @@ -35,7 +35,7 @@ def _audiox_server_cases(model: str): ] -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _audiox_server_cases(AUDIOX_TEST_MODEL), indirect=True) def test_audiox_t2a_online(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: diff --git a/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py b/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py index e770baf56..c9e7362fc 100644 --- a/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py +++ b/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py @@ -18,7 +18,7 @@ from tests.helpers.media import load_test_audio_data_url from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -pytestmark = [pytest.mark.full_model, pytest.mark.tts] +pytestmark = [pytest.mark.slow, pytest.mark.tts] MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512" @@ -66,6 +66,7 @@ tts_async_chunk_server_params = [ @hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4644") @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_clone_zh_001(omni_server, openai_client) -> None: """ @@ -88,6 +89,7 @@ def test_voice_clone_zh_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4644") @pytest.mark.parametrize("omni_server", tts_async_chunk_server_params, indirect=True) def test_voice_clone_zh_002(omni_server, openai_client) -> None: """ @@ -110,6 +112,7 @@ def test_voice_clone_zh_002(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4644") @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_clone_en_001(omni_server, openai_client) -> None: """ diff --git a/tests/e2e/online_serving/test_covo_audio_expansion.py b/tests/e2e/online_serving/test_covo_audio_expansion.py index a1baca554..33771d647 100644 --- a/tests/e2e/online_serving/test_covo_audio_expansion.py +++ b/tests/e2e/online_serving/test_covo_audio_expansion.py @@ -33,7 +33,7 @@ test_params = [ ] -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "H100"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) diff --git a/tests/e2e/online_serving/test_dreamzero_expansion.py b/tests/e2e/online_serving/test_dreamzero_expansion.py index 79429c123..38e7d4b16 100644 --- a/tests/e2e/online_serving/test_dreamzero_expansion.py +++ b/tests/e2e/online_serving/test_dreamzero_expansion.py @@ -60,7 +60,7 @@ def _write_synthetic_dreamzero_videos(client_mod, video_dir: Path) -> None: _write_synthetic_video(video_dir / file_name, client_mod.cv2, channel=channel) -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.distributed_cuda @hardware_test(res={"cuda": "H100"}, num_cards=2) diff --git a/tests/e2e/online_serving/test_dynin_omni_expansion.py b/tests/e2e/online_serving/test_dynin_omni_expansion.py index 702ff7f94..348191b6e 100644 --- a/tests/e2e/online_serving/test_dynin_omni_expansion.py +++ b/tests/e2e/online_serving/test_dynin_omni_expansion.py @@ -15,7 +15,7 @@ from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -pytestmark = [pytest.mark.full_model, pytest.mark.omni] +pytestmark = [pytest.mark.slow, pytest.mark.omni] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" diff --git a/tests/e2e/online_serving/test_ernie_image_expansion.py b/tests/e2e/online_serving/test_ernie_image_expansion.py index 1f2447677..848fec4e6 100644 --- a/tests/e2e/online_serving/test_ernie_image_expansion.py +++ b/tests/e2e/online_serving/test_ernie_image_expansion.py @@ -38,7 +38,7 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", diff --git a/tests/e2e/online_serving/test_flux2_expansion.py b/tests/e2e/online_serving/test_flux2_expansion.py index c900b91e8..2659cb927 100644 --- a/tests/e2e/online_serving/test_flux2_expansion.py +++ b/tests/e2e/online_serving/test_flux2_expansion.py @@ -19,7 +19,7 @@ from tests.helpers.runtime import ( dummy_messages_from_mix_data, ) -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4) POSITIVE_PROMPT = "A cat sitting on a windowsill" diff --git a/tests/e2e/online_serving/test_flux_2_dev_expansion.py b/tests/e2e/online_serving/test_flux_2_dev_expansion.py index aaf118112..77a7a9271 100644 --- a/tests/e2e/online_serving/test_flux_2_dev_expansion.py +++ b/tests/e2e/online_serving/test_flux_2_dev_expansion.py @@ -19,7 +19,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "black-forest-labs/FLUX.2-dev" PROMPT = "A cinematic mountain landscape at sunrise, dramatic clouds, ultra-detailed, realistic photography." diff --git a/tests/e2e/online_serving/test_flux_kontext_expansion.py b/tests/e2e/online_serving/test_flux_kontext_expansion.py index fd7d7d1b4..5fa4eb2a1 100644 --- a/tests/e2e/online_serving/test_flux_kontext_expansion.py +++ b/tests/e2e/online_serving/test_flux_kontext_expansion.py @@ -8,7 +8,7 @@ import pytest from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist" diff --git a/tests/e2e/online_serving/test_glm_tts.py b/tests/e2e/online_serving/test_glm_tts_expansion.py similarity index 74% rename from tests/e2e/online_serving/test_glm_tts.py rename to tests/e2e/online_serving/test_glm_tts_expansion.py index 7c6ad9764..db3423732 100644 --- a/tests/e2e/online_serving/test_glm_tts.py +++ b/tests/e2e/online_serving/test_glm_tts_expansion.py @@ -1,10 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -E2E Online tests for GLM-TTS model with text input and audio output. +E2E expansion tests for GLM-TTS online serving (nightly CI). -These tests verify the /v1/audio/speech endpoint works correctly with -the GLM-TTS two-stage pipeline (AR + DiT). +These tests verify the /v1/audio/speech endpoint with the GLM-TTS two-stage pipeline. """ import os @@ -19,16 +18,14 @@ from tests.helpers.media import load_test_audio_data_url from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path +pytestmark = [pytest.mark.slow, pytest.mark.tts] + MODEL = os.environ.get("GLM_TTS_MODEL_PATH", "zai-org/GLM-TTS") REF_TEXT = "他当时还跟线下其他的站姐吵架，然后，打架进局子了。" DEPLOY_CONFIG = get_deploy_config_path("glm_tts.yaml") - -# Official GLM-TTS reference audio and transcript. Vendored under tests/assets/ -# so Buildkite does not depend on raw.githubusercontent.com during E2E. REF_AUDIO_URL = load_test_audio_data_url("glm_tts/jiayan_zh.wav") - SYNC_EXTRA_ARGS = [ "--trust-remote-code", "--disable-log-stats", @@ -65,16 +62,7 @@ tts_async_chunk_server_params = [ ] -# GLM-TTS is a zero-shot voice cloning model (always requires ref_audio). -# There is no text-only / non-clone path — the official implementation -# asserts prompt_feat.shape[1] != 0 in token2wav_with_cache and every -# example prompt includes a reference audio. CosyVoice3 follows the same -# pattern: all tests are voice_clone_*. - - -@pytest.mark.advanced_model -@pytest.mark.omni -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_sync_server_params, indirect=True) def test_voice_clone_zh_sync(omni_server, openai_client) -> None: """ @@ -83,9 +71,6 @@ def test_voice_clone_zh_sync(omni_server, openai_client) -> None: Input Modal: text + ref_audio + ref_text Output Modal: audio Input Setting: stream=False - - Uses the official jiayan_zh.wav reference audio from the upstream - GLM-TTS repository (real human speech matching REF_TEXT). """ request_config = { "model": omni_server.model, @@ -98,9 +83,7 @@ def test_voice_clone_zh_sync(omni_server, openai_client) -> None: openai_client.send_audio_speech_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_async_chunk_server_params, indirect=True) def test_voice_clone_zh_async_chunk(omni_server, openai_client) -> None: """ @@ -121,9 +104,7 @@ def test_voice_clone_zh_async_chunk(omni_server, openai_client) -> None: openai_client.send_audio_speech_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_sync_server_params, indirect=True) def test_models_endpoint(omni_server, openai_client) -> None: """Test the /v1/models endpoint returns loaded model.""" diff --git a/tests/e2e/online_serving/test_hidream_i1_full_expansion.py b/tests/e2e/online_serving/test_hidream_i1_full_expansion.py index f84cc08ad..a362d4902 100644 --- a/tests/e2e/online_serving/test_hidream_i1_full_expansion.py +++ b/tests/e2e/online_serving/test_hidream_i1_full_expansion.py @@ -14,7 +14,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "HiDream-ai/HiDream-I1-Full" PROMPT = "A cinematic mountain landscape at sunrise, dramatic clouds, ultra-detailed, realistic photography." @@ -46,6 +46,7 @@ def _get_hidream_i1_image_feature_cases(model: str): _get_hidream_i1_image_feature_cases(MODEL), indirect=True, ) +@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4662") def test_hidream_i1_image( omni_server: OmniServer, openai_client: OpenAIClientHandler, diff --git a/tests/e2e/online_serving/test_higgs_audio_v2.py b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py similarity index 89% rename from tests/e2e/online_serving/test_higgs_audio_v2.py rename to tests/e2e/online_serving/test_higgs_audio_v2_expansion.py index 92688ad0b..41f2b6a6a 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v2.py +++ b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -End-to-end online tests for higgs-audio v2 against /v1/audio/speech. +E2E Online expansion tests for higgs-audio v2 against /v1/audio/speech. v1 scope is plain text -> 24 kHz speech plus shallow voice clone via ref_audio + ref_text (inline) or voice= (after POST /v1/audio/voices). @@ -31,6 +31,8 @@ from tests.helpers.media import load_test_audio_data_url from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path +pytestmark = [pytest.mark.slow, pytest.mark.tts] + MODEL = "bosonai/higgs-audio-v2-generation-3B-base" STAGE_CONFIG = get_deploy_config_path("higgs_audio_v2.yaml") SERVER_ARGS = ["--trust-remote-code", "--disable-log-stats"] @@ -69,9 +71,7 @@ _REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know w class TestHiggsAudioV2OnlineHappyPath: """Plain-text -> audio happy paths against the live HTTP server.""" - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_wav(self, omni_server, openai_client) -> None: """Single non-streaming WAV request — covers the canonical TTS happy path.""" openai_client.send_audio_speech_request( @@ -85,9 +85,7 @@ class TestHiggsAudioV2OnlineHappyPath: } ) - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None: """Streaming PCM via the shared-memory connector's codec_streaming path. @@ -106,9 +104,7 @@ class TestHiggsAudioV2OnlineHappyPath: } ) - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None: """max_new_tokens is one of the few extra fields the higgs validator accepts.""" openai_client.send_audio_speech_request( @@ -123,9 +119,7 @@ class TestHiggsAudioV2OnlineHappyPath: } ) - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_concurrent_plain_text(self, omni_server, openai_client) -> None: """Three concurrent non-streaming requests — guards the per-slot audio state.""" openai_client.send_audio_speech_request( @@ -154,9 +148,7 @@ class TestHiggsAudioV2OnlineHappyPath: class TestHiggsAudioV2OnlineValidatorRejections: """Out-of-scope fields must come back as 4xx with a higgs-named message.""" - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_rejects_ref_audio_without_ref_text(self, omni_server, openai_client) -> None: """Voice clone needs the transcript too — half-supplied is a 4xx.""" openai_client.send_audio_speech_request( @@ -171,9 +163,7 @@ class TestHiggsAudioV2OnlineValidatorRejections: } ) - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_rejects_ref_text_without_ref_audio(self, omni_server, openai_client) -> None: """Symmetric guard: ref_text alone is not enough — must come with ref_audio.""" openai_client.send_audio_speech_request( @@ -188,9 +178,7 @@ class TestHiggsAudioV2OnlineValidatorRejections: } ) - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_rejects_task_type(self, omni_server, openai_client) -> None: openai_client.send_audio_speech_request( { @@ -204,9 +192,7 @@ class TestHiggsAudioV2OnlineValidatorRejections: } ) - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_rejects_language_override(self, omni_server, openai_client) -> None: openai_client.send_audio_speech_request( { @@ -220,9 +206,7 @@ class TestHiggsAudioV2OnlineValidatorRejections: } ) - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_rejects_multi_speaker_tag_in_text(self, omni_server, openai_client) -> None: openai_client.send_audio_speech_request( { @@ -235,9 +219,7 @@ class TestHiggsAudioV2OnlineValidatorRejections: } ) - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_rejects_empty_input(self, omni_server, openai_client) -> None: openai_client.send_audio_speech_request( { @@ -262,9 +244,7 @@ class TestHiggsAudioV2OnlineVoiceClone: :meth:`HiggsAudioV2TalkerForConditionalGeneration._maybe_apply_ref_audio_substitution`. """ - @pytest.mark.advanced_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voice_clone_basic(self, omni_server, openai_client) -> None: """Single non-streaming WAV request driven by the qwen3_tts ref clip.""" openai_client.send_audio_speech_request( @@ -280,9 +260,7 @@ class TestHiggsAudioV2OnlineVoiceClone: } ) - @pytest.mark.advanced_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voice_clone_pcm_streaming(self, omni_server, openai_client) -> None: """Voice clone over the streaming PCM path.""" openai_client.send_audio_speech_request( diff --git a/tests/e2e/online_serving/test_higgs_audio_v3.py b/tests/e2e/online_serving/test_higgs_audio_v3.py index d6aecce41..b8c37a211 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v3.py +++ b/tests/e2e/online_serving/test_higgs_audio_v3.py @@ -60,8 +60,9 @@ class TestHiggsAudioV3OnlineHappyPath: """Plain-text -> audio happy paths against the live HTTP server.""" @pytest.mark.core_model + @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_wav(self, omni_server, openai_client) -> None: """Single non-streaming WAV request - canonical TTS happy path.""" openai_client.send_audio_speech_request( @@ -71,13 +72,12 @@ class TestHiggsAudioV3OnlineHappyPath: "stream": False, "response_format": "wav", "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "min_audio_bytes": _MIN_AUDIO_BYTES, } ) @pytest.mark.core_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None: """``max_new_tokens`` is one of the few extra fields the v3 validator accepts.""" openai_client.send_audio_speech_request( @@ -87,31 +87,33 @@ class TestHiggsAudioV3OnlineHappyPath: "stream": False, "response_format": "wav", "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "max_new_tokens": 500, + "max_new_tokens": 32, "min_audio_bytes": _MIN_AUDIO_BYTES, } ) + @pytest.mark.skip(reason="issue#4411") @pytest.mark.core_model + @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_concurrent_plain_text(self, omni_server, openai_client) -> None: """Three concurrent non-streaming requests - guards per-slot audio state and Stage-0 PC under batching.""" openai_client.send_audio_speech_request( { "model": omni_server.model, - "input": "It was the night before my birthday.", + "input": "it was the night before my birthday", "stream": False, "response_format": "wav", "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "min_audio_bytes": _MIN_AUDIO_BYTES, }, request_num=3, ) - @pytest.mark.core_model + @pytest.mark.skip(reason="issue#4411") + @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None: """Streaming PCM via talker2code2wav_async_chunk + the connector's codec_streaming path. @@ -135,15 +137,14 @@ class TestHiggsAudioV3OnlineHappyPath: "stream": True, "response_format": "pcm", "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "min_audio_bytes": _MIN_AUDIO_BYTES, "min_hnr_db": 0.0, } ) @pytest.mark.skip(reason="issue#4411") - @pytest.mark.core_model + @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_concurrent_pcm_streaming(self, omni_server, openai_client) -> None: """Three concurrent streaming requests - guards per-request frame cursors in ``talker2code2wav_async_chunk`` and per-slot delay-pattern state under batched AR. @@ -161,7 +162,6 @@ class TestHiggsAudioV3OnlineHappyPath: "stream": True, "response_format": "pcm", "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "min_audio_bytes": _MIN_AUDIO_BYTES, "min_hnr_db": -2.0, }, request_num=3, @@ -185,7 +185,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: @pytest.mark.core_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_emotion_and_expressive(self, omni_server, openai_client) -> None: """Delivery tokens (emotion + expressive_high) at the start of input.""" openai_client.send_audio_speech_request( @@ -201,7 +201,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: @pytest.mark.core_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_style_whispering(self, omni_server, openai_client) -> None: """Style token at the start - ``whispering``.""" openai_client.send_audio_speech_request( @@ -217,7 +217,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: @pytest.mark.core_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_prosody_speed_and_pitch(self, omni_server, openai_client) -> None: """Two prosody tokens at the start - slow speed plus low pitch.""" openai_client.send_audio_speech_request( @@ -235,7 +235,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: @pytest.mark.core_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_pause_mid_text(self, omni_server, openai_client) -> None: """Positional pause token placed inline between two clauses.""" openai_client.send_audio_speech_request( @@ -251,7 +251,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: @pytest.mark.core_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_sfx_with_onomatopoeia(self, omni_server, openai_client) -> None: """SFX token paired with its written onomatopoeia (``<|sfx:laughter|>Hehe``).""" openai_client.send_audio_speech_request( @@ -274,8 +274,9 @@ class TestHiggsAudioV3OnlineVoiceClone: """Voice clone via the two payload shapes the model serves.""" @pytest.mark.core_model + @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voice_clone_ref_audio_ref_text(self, omni_server, openai_client) -> None: """Canonical vllm-omni voice clone via ``ref_audio`` + ``ref_text``.""" openai_client.send_audio_speech_request( @@ -287,13 +288,12 @@ class TestHiggsAudioV3OnlineVoiceClone: "stream": False, "response_format": "wav", "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "min_audio_bytes": _MIN_AUDIO_BYTES, } ) @pytest.mark.core_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voice_clone_references_alias(self, omni_server, openai_client) -> None: """BosonAI cookbook payload: ``references=[{audio_path, text}]``. @@ -309,81 +309,5 @@ class TestHiggsAudioV3OnlineVoiceClone: "stream": False, "response_format": "wav", "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "min_audio_bytes": _MIN_AUDIO_BYTES, - } - ) - - -@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) -class TestHiggsAudioV3OnlineValidatorRejections: - """Out-of-scope shapes must come back as 4xx. - - These cases use ``send_audio_speech_http_request`` (raw HTTP POST against - ``/v1/audio/speech``) rather than the OpenAI SDK helper because: - - - We need the failure-path matcher; the SDK helper asserts success. - - The cookbook ``references`` field is not in the OpenAI SDK schema, so - the SDK strips it before the request leaves the client and the server - sees a plain TTS request that returns 200. Raw HTTP keeps the JSON body - intact so the validator actually fires. - """ - - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_rejects_empty_input(self, omni_server, openai_client) -> None: - """Empty text input - validator should reject before reaching the engine.""" - openai_client.send_audio_speech_http_request( - { - "json": { - "model": omni_server.model, - "input": "", - "response_format": "wav", - }, - "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "err_code": (400, 422), - "err_message": ("input", "empty"), - } - ) - - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_rejects_multi_reference_payload(self, omni_server, openai_client) -> None: - """``references[]`` with more than one entry - multi-shot voice clone is not supported.""" - openai_client.send_audio_speech_http_request( - { - "json": { - "model": omni_server.model, - "input": "Hello world.", - "references": [ - {"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT}, - {"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT}, - ], - "response_format": "wav", - }, - "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "err_code": (400, 422), - "err_message": "references", - } - ) - - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_rejects_conflicting_ref_audio_and_references(self, omni_server, openai_client) -> None: - """``ref_audio`` and ``references`` cannot both be set to different values.""" - openai_client.send_audio_speech_http_request( - { - "json": { - "model": omni_server.model, - "input": "Hello world.", - "ref_audio": _REF_AUDIO_URL, - "references": [{"audio_path": "https://example.com/other.wav"}], - "response_format": "wav", - }, - "timeout": DEFAULT_SPEECH_TIMEOUT_S, - "err_code": (400, 422), - "err_message": "mutually exclusive", } ) diff --git a/tests/e2e/online_serving/test_hunyuan_video_15.py b/tests/e2e/online_serving/test_hunyuan_video_15.py deleted file mode 100644 index daa3fe6d8..000000000 --- a/tests/e2e/online_serving/test_hunyuan_video_15.py +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests of common diffusion feature combinations in online serving mode -for HunyuanVideo-1.5-T2V (480p). - -Coverage (H100, since model cannot fit L4): -- CacheDiT + Layerwise CPU offloading (1 GPU) -- CacheDiT + TP=2 + VAE patch parallel=2 (2 GPUs) -""" - -import pytest - -from tests.helpers.mark import hardware_marks -from tests.helpers.runtime import ( - OmniServer, - OmniServerParams, - OpenAIClientHandler, -) - -PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion." -NEGATIVE_PROMPT = "low quality, blurry, distorted" - -MODEL = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" - -SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) - - -def _get_diffusion_feature_cases(model: str): - """Return L4 diffusion feature cases for HunyuanVideo-1.5. - - Designed for 2x H100 environment per issue #1832. - """ - return [ - # (1 GPU) CacheDiT + Layerwise CPU offloading - pytest.param( - OmniServerParams( - model=model, - ), - id="default", - marks=SINGLE_CARD_MARKS, - ), - ] - - -@pytest.mark.advanced_model -@pytest.mark.core_model -@pytest.mark.diffusion -@pytest.mark.parametrize( - "omni_server", - _get_diffusion_feature_cases(MODEL), - indirect=True, -) -def test_text_to_video_001( - omni_server: OmniServer, - openai_client: OpenAIClientHandler, -): - """L4 diffusion feature coverage for HunyuanVideo-1.5-T2V on H100.""" - form_data = { - "prompt": PROMPT, - "negative_prompt": NEGATIVE_PROMPT, - "height": 480, - "width": 640, - "num_frames": 5, - "num_inference_steps": 2, - "guidance_scale": 6.0, - "seed": 42, - } - - request_config = { - "model": omni_server.model, - "form_data": form_data, - } - - openai_client.send_video_diffusion_request(request_config) diff --git a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py index 0999ea8b5..a448335c6 100644 --- a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py +++ b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py @@ -14,7 +14,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion." NEGATIVE_PROMPT = "low quality, blurry, distorted" diff --git a/tests/e2e/online_serving/test_longcat_image.py b/tests/e2e/online_serving/test_longcat_image.py deleted file mode 100644 index 9afe07192..000000000 --- a/tests/e2e/online_serving/test_longcat_image.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Recommended tests of diffusion features that are available in online serving mode -and are supported by the following model: -- LongCat-Image: text-to-image with single prompt input -Coverage: -- CPU offloading (model-level sequential offload via --enable-cpu-offload) -- Cache-DiT -- SP (Ulysses) - -This validates: - - Successful image generation at the expected 768x1344 resolution with recommended feature combinations -""" - -import pytest - -from tests.helpers.mark import hardware_marks -from tests.helpers.runtime import ( - OmniServer, - OmniServerParams, - OpenAIClientHandler, - dummy_messages_from_mix_data, -) - -TEXT_TO_IMAGE_PROMPT = ( - "A cinematic illustration of a cat typing on a silver laptop, soft window light, highly detailed." -) -NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated" -SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) - - -def _get_diffusion_feature_cases(model: str): - """Return diffusion feature cases for LongCat-Image.""" - return [ - pytest.param( - OmniServerParams( - model=model, - ), - id="default", - marks=SINGLE_CARD_FEATURE_MARKS, - ) - ] - - -@pytest.mark.advanced_model -@pytest.mark.core_model -@pytest.mark.diffusion -@pytest.mark.parametrize( - "omni_server", - _get_diffusion_feature_cases("meituan-longcat/LongCat-Image"), - indirect=True, -) -def test_longcat_image(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Test the recommended feature combinations for LongCat-Image.""" - messages = dummy_messages_from_mix_data(content_text=TEXT_TO_IMAGE_PROMPT) - - request_config = { - "model": omni_server.model, - "messages": messages, - "extra_body": { - "height": 768, - "width": 1344, - "num_inference_steps": 2, - "negative_prompt": NEGATIVE_PROMPT, - "guidance_scale": 4.0, - "seed": 42, - }, - } - - openai_client.send_diffusion_request(request_config) diff --git a/tests/e2e/online_serving/test_longcat_image_edit.py b/tests/e2e/online_serving/test_longcat_image_edit.py deleted file mode 100644 index a59186cbd..000000000 --- a/tests/e2e/online_serving/test_longcat_image_edit.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Recommended tests of diffusion features that are available in online serving mode -and are supported by the following model: -- LongCat-Image-Edit: image-to-image edit with a single image + single edit prompt input -Coverage: -- CPU offloading (model-level sequential offload via --enable-cpu-offload) -- Cache-DiT -- SP (Ulysses) - -This validates: - - Successful image generation at the expected 1024x1024 resolution with recommended feature combinations -""" - -import pytest - -from tests.helpers.mark import hardware_marks -from tests.helpers.media import generate_synthetic_image -from tests.helpers.runtime import ( - OmniServer, - OmniServerParams, - OpenAIClientHandler, - dummy_messages_from_mix_data, -) - -EDIT_PROMPT = "Transform this modern image into a cinematic animation style with vibrant colors and soft lighting." -NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated" -SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) - - -def _get_diffusion_feature_cases(model: str): - """Return diffusion feature cases for LongCat-Image-Edit.""" - return [ - pytest.param( - OmniServerParams( - model=model, - ), - id="default", - marks=SINGLE_CARD_FEATURE_MARKS, - ) - ] - - -@pytest.mark.advanced_model -@pytest.mark.core_model -@pytest.mark.diffusion -@pytest.mark.parametrize( - "omni_server", - _get_diffusion_feature_cases("meituan-longcat/LongCat-Image-Edit"), - indirect=True, -) -def test_longcat_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Test the recommended feature combinations for LongCat-Image-Edit.""" - image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" - messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) - - request_config = { - "model": omni_server.model, - "messages": messages, - "extra_body": { - "height": 1024, - "width": 1024, - "num_inference_steps": 2, - "negative_prompt": NEGATIVE_PROMPT, - "guidance_scale": 4.0, - "seed": 42, - }, - } - - openai_client.send_diffusion_request(request_config) diff --git a/tests/e2e/online_serving/test_longcat_image_edit_expansion.py b/tests/e2e/online_serving/test_longcat_image_edit_expansion.py index 4be5fe42d..7c7e0788c 100644 --- a/tests/e2e/online_serving/test_longcat_image_edit_expansion.py +++ b/tests/e2e/online_serving/test_longcat_image_edit_expansion.py @@ -3,6 +3,7 @@ Recommended tests of diffusion features that are available in online serving mod and are supported by the following model: - LongCat-Image-Edit: image-to-image edit with a single image + single edit prompt input Coverage: +- Default smoke (1 GPU) - CPU offloading (model-level sequential offload via --enable-cpu-offload) - Cache-DiT - SP (Ulysses) @@ -17,7 +18,7 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] EDIT_PROMPT = "Transform this modern image into a cinematic animation style with vibrant colors and soft lighting." NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated" @@ -25,9 +26,17 @@ SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) +MODEL = "meituan-longcat/LongCat-Image-Edit" + + def _get_diffusion_feature_cases(model: str): """Return diffusion feature cases for LongCat-Image-Edit.""" return [ + pytest.param( + OmniServerParams(model=model), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), pytest.param( OmniServerParams( model=model, @@ -54,7 +63,7 @@ def _get_diffusion_feature_cases(model: str): @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("meituan-longcat/LongCat-Image-Edit"), + _get_diffusion_feature_cases(MODEL), indirect=True, ) def test_longcat_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_longcat_image_expansion.py b/tests/e2e/online_serving/test_longcat_image_expansion.py index b9fd858b0..5ed7bb156 100644 --- a/tests/e2e/online_serving/test_longcat_image_expansion.py +++ b/tests/e2e/online_serving/test_longcat_image_expansion.py @@ -3,6 +3,7 @@ Recommended tests of diffusion features that are available in online serving mod and are supported by the following model: - LongCat-Image: text-to-image with single prompt input Coverage: +- Default smoke (1 GPU) - CPU offloading (model-level sequential offload via --enable-cpu-offload) - Cache-DiT - SP (Ulysses) @@ -16,7 +17,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] TEXT_TO_IMAGE_PROMPT = ( "A cinematic illustration of a cat typing on a silver laptop, soft window light, highly detailed." @@ -25,10 +26,17 @@ NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated" SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) +MODEL = "meituan-longcat/LongCat-Image" + def _get_diffusion_feature_cases(model: str): """Return diffusion feature cases for LongCat-Image.""" return [ + pytest.param( + OmniServerParams(model=model), + id="default", + marks=SINGLE_CARD_FEATURE_MARKS, + ), pytest.param( OmniServerParams( model=model, @@ -55,7 +63,7 @@ def _get_diffusion_feature_cases(model: str): @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("meituan-longcat/LongCat-Image"), + _get_diffusion_feature_cases(MODEL), indirect=True, ) def test_longcat_image(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_mimo_audio.py b/tests/e2e/online_serving/test_mimo_audio_expansion.py similarity index 82% rename from tests/e2e/online_serving/test_mimo_audio.py rename to tests/e2e/online_serving/test_mimo_audio_expansion.py index f65d099c6..e5722ddb8 100644 --- a/tests/e2e/online_serving/test_mimo_audio.py +++ b/tests/e2e/online_serving/test_mimo_audio_expansion.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -E2E Online tests for MiMo-Audio model with audio/text input and audio output. +E2E expansion tests for MiMo-Audio online serving (nightly Omni · Function Test with H100). """ import os @@ -17,6 +17,8 @@ from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) +pytestmark = [pytest.mark.slow, pytest.mark.omni] + MIMO_AUDIO_TOKENIZER_REPO = "XiaomiMiMo/MiMo-Audio-Tokenizer" CHAT_TEMPLATE_PATH = str( Path(__file__).parent.parent.parent.parent / "examples" / "online_serving" / "mimo_audio" / "chat_template.jinja" @@ -39,16 +41,11 @@ def download_tokenizer(): return local_path -# Guard module-level setup so test collection doesn't fail in environments -# where the model cache is read-only or models aren't available. try: stage_configs = [get_deploy_config_path("mimo_audio.yaml")] tokenizer_path = download_tokenizer() os.environ["MIMO_AUDIO_TOKENIZER_PATH"] = tokenizer_path - # --load-format dummy applies to every stage pipeline-wide, avoiding a - # per-stage yaml rewrite (the old approach wrote a tempfile + atexit-unlink - # which raced with CI's process lifecycle). test_params = [ OmniServerParams( model=model, @@ -60,7 +57,7 @@ try: ] except Exception as exc: pytest.skip( - f"MiMo-Audio online serving tests skipped: module setup failed ({type(exc).__name__}: {exc})", + f"MiMo-Audio expansion tests skipped: module setup failed ({type(exc).__name__}: {exc})", allow_module_level=True, ) @@ -78,45 +75,7 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.advanced_model -@pytest.mark.core_model -@pytest.mark.omni -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1) -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -@pytest.mark.skip(reason="CI failed 8571") -def test_audio_to_text_audio_001(omni_server, openai_client) -> None: - """ - Test audio and text input processing and text/audio output generation via OpenAI API. - Deploy Setting: default yaml - Input Modal: text + audio - Output Modal: text + audio - Input Setting: stream=True - Datasets: single request - """ - - audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1, sample_rate=24000)['base64']}" - messages = dummy_messages_from_mix_data( - audio_data_url=audio_data_url, - content_text=get_prompt("audio"), - ) - - request_config = { - "model": omni_server.model, - "messages": messages, - "stream": True, - "sampling_params_list": [{"max_tokens": 64}, {"max_tokens": 64}], - "key_words": { - "audio": ["test"], - }, - } - - # Test single completion - openai_client.send_omni_request(request_config) - - -@pytest.mark.advanced_model -@pytest.mark.omni -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1) +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_001(omni_server, openai_client) -> None: """ @@ -137,3 +96,34 @@ def test_text_to_text_001(omni_server, openai_client) -> None: } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.skip(reason="CI failed 8571") +def test_audio_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test audio and text input processing and text/audio output generation via OpenAI API. + Deploy Setting: default yaml + Input Modal: text + audio + Output Modal: text + audio + Input Setting: stream=True + Datasets: single request + """ + audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1, sample_rate=24000)['base64']}" + messages = dummy_messages_from_mix_data( + audio_data_url=audio_data_url, + content_text=get_prompt("audio"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + "sampling_params_list": [{"max_tokens": 64}, {"max_tokens": 64}], + "key_words": { + "audio": ["test"], + }, + } + + openai_client.send_omni_request(request_config) diff --git a/tests/e2e/online_serving/test_ming_tts.py b/tests/e2e/online_serving/test_ming_tts_expansion.py similarity index 92% rename from tests/e2e/online_serving/test_ming_tts.py rename to tests/e2e/online_serving/test_ming_tts_expansion.py index 03982cc4a..40619617b 100644 --- a/tests/e2e/online_serving/test_ming_tts.py +++ b/tests/e2e/online_serving/test_ming_tts_expansion.py @@ -1,21 +1,25 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -E2E online serving tests for Ming-omni-tts (dense) model. +E2E expansion tests for Ming-omni-tts online serving (nightly CI). + Tests text-to-audio via /v1/audio/speech endpoint. """ import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - import pytest from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -pytestmark = [pytest.mark.advanced_model, pytest.mark.omni] +pytestmark = [ + pytest.mark.slow, + pytest.mark.tts, + pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4704"), +] MODEL = "inclusionAI/Ming-omni-tts-0.5B" DEPLOY_CONFIG = get_deploy_config_path("ming_tts.yaml") diff --git a/tests/e2e/online_serving/test_moss_tts_expansion.py b/tests/e2e/online_serving/test_moss_tts_expansion.py index 80ee5f360..4aae8c7c0 100644 --- a/tests/e2e/online_serving/test_moss_tts_expansion.py +++ b/tests/e2e/online_serving/test_moss_tts_expansion.py @@ -3,8 +3,8 @@ """ E2E online tests for the full MOSS-TTS family via /v1/audio/speech. -The offline suite (``tests/e2e/offline_inference/test_moss_tts.py`` and -``test_moss_tts_v1_5.py``) covers the engine path; this file covers the +The offline suite (``tests/e2e/offline_inference/test_moss_tts_*_expansion.py``) +covers the engine path; this file covers the *serving* path (``serving_speech.py`` → ``_detect_moss_variant`` → the delay/realtime prompt builders), which the Nano online test does not exercise. @@ -30,11 +30,17 @@ from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -# TODO: Fix this test -# pytestmark = [pytest.mark.full_model, pytest.mark.tts] +pytestmark = [ + pytest.mark.slow, + pytest.mark.tts, + pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"), +] MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime" REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav" +# Voice-clone output is not reliably transcribed by the Whisper check used at +# full_model run_level; assert non-trivial WAV payload size instead. +_MIN_AUDIO_BYTES = 10_000 @pytest.fixture(scope="session") @@ -94,6 +100,10 @@ def test_text_to_audio_001(omni_server, openai_client, ref_audio_data_url) -> No Output Modal: audio (24 kHz, WAV) Input Setting: stream=False Datasets: single request + + NOTE: ``min_audio_bytes`` skips Whisper transcript similarity — cloned + voice output is often empty or mismatched under ASR without indicating a + real serving regression. """ request_config = { "model": omni_server.model, @@ -101,6 +111,7 @@ def test_text_to_audio_001(omni_server, openai_client, ref_audio_data_url) -> No "stream": False, "response_format": "wav", "ref_audio": ref_audio_data_url, + "min_audio_bytes": _MIN_AUDIO_BYTES, } openai_client.send_audio_speech_request(request_config) @@ -144,6 +155,8 @@ def test_text_to_audio_003_chinese(omni_server, openai_client, ref_audio_data_ur Output Modal: audio (24 kHz, WAV) Input Setting: stream=False Datasets: single request + + NOTE: same ``min_audio_bytes`` rationale as test_text_to_audio_001. """ request_config = { "model": omni_server.model, @@ -151,6 +164,7 @@ def test_text_to_audio_003_chinese(omni_server, openai_client, ref_audio_data_ur "stream": False, "response_format": "wav", "ref_audio": ref_audio_data_url, + "min_audio_bytes": _MIN_AUDIO_BYTES, } openai_client.send_audio_speech_request(request_config) diff --git a/tests/e2e/online_serving/test_moss_tts_nano_expansion.py b/tests/e2e/online_serving/test_moss_tts_nano_expansion.py index abcb900d1..4b1a99b2c 100644 --- a/tests/e2e/online_serving/test_moss_tts_nano_expansion.py +++ b/tests/e2e/online_serving/test_moss_tts_nano_expansion.py @@ -24,9 +24,9 @@ from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -# ``omni`` for all tests; 001/002 also get ``full_model`` via ``TestMossTtsNanoFull`` (003 is core+advanced only, no full_model). +# ``omni`` for all tests; 001/002 also get ``slow`` via ``TestMossTtsNanoFull`` (003 is core+advanced only, no slow). pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.tts, ] diff --git a/tests/e2e/online_serving/test_nextstep_expansion.py b/tests/e2e/online_serving/test_nextstep_expansion.py index 24384d29c..381079320 100644 --- a/tests/e2e/online_serving/test_nextstep_expansion.py +++ b/tests/e2e/online_serving/test_nextstep_expansion.py @@ -14,7 +14,7 @@ from tests.helpers.runtime import ( dummy_messages_from_mix_data, ) -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.slow, pytest.mark.diffusion] # L4: 4 GPUs + TP=4; XPU B60: 2 cards (use num_cards={"cuda": 4, "xpu": 4} if needed) FOUR_CARD_MARKS = hardware_marks( diff --git a/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py b/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py index c49b42aa7..bbf9ca2b6 100644 --- a/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py @@ -55,7 +55,7 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) @@ -93,7 +93,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.omni @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_001(omni_server, openai_client) -> None: diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py index a03e7ad8b..50494fb72 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py @@ -15,10 +15,15 @@ import pytest from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams -from tests.helpers.stage_config import get_deploy_config_path +from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" +_STAGE_CONFIG = modify_stage_config( + get_deploy_config_path("qwen3_tts.yaml"), + updates={"stages": {0: {"default_sampling_params.max_tokens": 500}}}, +) + def get_prompt(prompt_type="text"): """Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case).""" @@ -38,7 +43,7 @@ tts_server_params = [ pytest.param( OmniServerParams( model=MODEL, - stage_config_path=get_deploy_config_path("qwen3_tts.yaml"), + stage_config_path=_STAGE_CONFIG, server_args=["--trust-remote-code"], ), id="async_chunk", diff --git a/tests/e2e/online_serving/test_sd3.py b/tests/e2e/online_serving/test_sd3.py deleted file mode 100644 index 538e1bda8..000000000 --- a/tests/e2e/online_serving/test_sd3.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Tests for Stable Diffusion 3.5 medium model. -""" - -import pytest - -from tests.helpers.mark import hardware_marks -from tests.helpers.runtime import ( - OmniServer, - OmniServerParams, - OpenAIClientHandler, - dummy_messages_from_mix_data, -) - -FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4) -POSITIVE_PROMPT = "A serene mountain landscape at sunset" -NEGATIVE_PROMPT = "blurry, low quality, distorted" - - -# For now, we only test one or two good configurations for this model to keep -# the CI light. The test cases focus on the most common feature combinations -# that provide good performance improvements. -def _get_diffusion_feature_cases(model: str): - return [ - # Cache-DiT + CFG Parallel + Tensor Parallel - pytest.param( - OmniServerParams( - model=model, - ), - id="default", - marks=FOUR_CARD_FEATURE_MARKS, - ), - ] - - -@pytest.mark.advanced_model -@pytest.mark.core_model -@pytest.mark.diffusion -@pytest.mark.parametrize( - "omni_server", - _get_diffusion_feature_cases( - model="stabilityai/stable-diffusion-3.5-medium", - ), - indirect=True, -) -def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): - messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT) - request_config = { - "model": omni_server.model, - "messages": messages, - "extra_body": { - "height": 1024, - "width": 1024, - "num_inference_steps": 28, - "negative_prompt": NEGATIVE_PROMPT, - "guidance_scale": 4.5, - "seed": 42, - }, - } - - openai_client.send_diffusion_request(request_config) diff --git a/tests/e2e/online_serving/test_sd3_expansion.py b/tests/e2e/online_serving/test_sd3_expansion.py index c299ae8f1..f881a212b 100644 --- a/tests/e2e/online_serving/test_sd3_expansion.py +++ b/tests/e2e/online_serving/test_sd3_expansion.py @@ -12,7 +12,7 @@ from tests.helpers.runtime import ( dummy_messages_from_mix_data, ) -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4) POSITIVE_PROMPT = "A serene mountain landscape at sunset" diff --git a/tests/e2e/online_serving/test_step_audio2.py b/tests/e2e/online_serving/test_step_audio2.py deleted file mode 100644 index a23aaa0ff..000000000 --- a/tests/e2e/online_serving/test_step_audio2.py +++ /dev/null @@ -1,368 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -""" -E2E tests for Step-Audio2 online serving with concurrent requests. - -Tests: -1. Single request via API -2. Concurrent throughput (1/2/4/8 concurrent requests) -3. Latency percentiles (p50/p95/p99) -""" - -import asyncio -import os -import socket -import subprocess -import sys -import time -from pathlib import Path - -import aiohttp -import numpy as np -import pytest -from vllm.utils.network_utils import get_open_port - -MODEL = "stepfun-ai/Step-Audio-2-mini" -STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "step_audio2_ci.yaml") - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - - -class OmniServer: - """Omniserver for vLLM-Omni tests.""" - - def __init__( - self, - model: str, - serve_args: list[str], - *, - env_dict: dict[str, str] | None = None, - ) -> None: - self.model = model - self.serve_args = serve_args - self.env_dict = env_dict - self.proc: subprocess.Popen | None = None - self.host = "127.0.0.1" - self.port = get_open_port() - - def _start_server(self) -> None: - """Start the vLLM-Omni server subprocess.""" - env = os.environ.copy() - env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - if self.env_dict is not None: - env.update(self.env_dict) - - cmd = [ - sys.executable, - "-m", - "vllm_omni.entrypoints.cli.main", - "serve", - self.model, - "--omni", - "--host", - self.host, - "--port", - str(self.port), - ] + self.serve_args - - print(f"Launching OmniServer with: {' '.join(cmd)}") - self.proc = subprocess.Popen( - cmd, - env=env, - cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - ) - - max_wait = 600 # 10 minutes - start_time = time.time() - while time.time() - start_time < max_wait: - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.settimeout(1) - result = sock.connect_ex((self.host, self.port)) - if result == 0: - print(f"Server ready on {self.host}:{self.port}") - return - except Exception: - pass - time.sleep(2) - - raise RuntimeError(f"Server failed to start within {max_wait} seconds") - - def __enter__(self): - self._start_server() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.proc: - self.proc.terminate() - try: - self.proc.wait(timeout=30) - except subprocess.TimeoutExpired: - self.proc.kill() - self.proc.wait() - - -def create_dummy_audio_base64(duration_sec: float = 5.0, sample_rate: int = 16000) -> str: - """Create dummy audio as base64.""" - import base64 - import io - import wave - - num_samples = int(duration_sec * sample_rate) - t = np.linspace(0, duration_sec, num_samples, dtype=np.float32) - audio = (0.5 * np.sin(2 * np.pi * 440 * t) * 32767).astype(np.int16) - - buffer = io.BytesIO() - with wave.open(buffer, "wb") as wav_file: - wav_file.setnchannels(1) - wav_file.setsampwidth(2) - wav_file.setframerate(sample_rate) - wav_file.writeframes(audio.tobytes()) - - return base64.b64encode(buffer.getvalue()).decode("utf-8") - - -def create_s2st_request(audio_base64: str) -> dict: - """Create S2ST request payload.""" - # Use audio_url format with base64 data URL - audio_url = f"data:audio/wav;base64,{audio_base64}" - return { - "model": MODEL, - "messages": [ - {"role": "system", "content": "请仔细聆听这段语音，然后复述其内容。"}, - { - "role": "user", - "content": [ - { - "type": "audio_url", - "audio_url": {"url": audio_url}, - }, - { - "type": "text", - "text": "", - }, - ], - }, - ], - "max_tokens": 512, - "temperature": 0.7, - "top_p": 0.9, - } - - -async def send_request(session: aiohttp.ClientSession, url: str, payload: dict) -> tuple[float, int, str]: - """Send single request and return (latency, num_tokens, response_text).""" - start = time.perf_counter() - async with session.post(url, json=payload) as response: - result = await response.json() - latency = time.perf_counter() - start - - if "choices" in result and len(result["choices"]) > 0: - text = result["choices"][0].get("message", {}).get("content", "") - tokens = result.get("usage", {}).get("completion_tokens", len(text.split())) - else: - text = str(result) - tokens = 0 - - return latency, tokens, text - - -async def benchmark_concurrent( - url: str, - payload: dict, - num_concurrent: int, - num_requests: int, - warmup: int = 2, -) -> dict: - """Run concurrent benchmark with true sustained concurrency. - - Uses semaphore to maintain exactly num_concurrent requests in flight at all times, - rather than batch-and-wait approach. Measures real wall clock time. - """ - semaphore = asyncio.Semaphore(num_concurrent) - - async def bounded_request(session: aiohttp.ClientSession): - async with semaphore: - return await send_request(session, url, payload) - - async with aiohttp.ClientSession() as session: - if warmup > 0: - print(f" Warming up ({warmup} requests)...", end=" ", flush=True) - for _ in range(warmup): - try: - await send_request(session, url, payload) - except Exception as e: - print(f"Warmup failed: {e}") - print("done") - - print(f" Running {num_requests} requests with concurrency={num_concurrent}...", end=" ", flush=True) - wall_start = time.perf_counter() - - tasks = [bounded_request(session) for _ in range(num_requests)] - results = await asyncio.gather(*tasks, return_exceptions=True) - - wall_time = time.perf_counter() - wall_start - print(f"done ({wall_time:.2f}s)") - - all_latencies = [] - all_tokens = [] - errors = 0 - - for result in results: - if isinstance(result, Exception): - print(f" Request failed: {result}") - errors += 1 - continue - latency, tokens, _ = result - all_latencies.append(latency) - all_tokens.append(tokens) - - if not all_latencies: - return {"error": "All requests failed"} - - latencies = np.array(all_latencies) - total_tokens = sum(all_tokens) - successful_requests = len(all_latencies) - - return { - "num_concurrent": num_concurrent, - "num_requests": num_requests, - "successful_requests": successful_requests, - "errors": errors, - "wall_time": wall_time, - "p50_latency": float(np.percentile(latencies, 50)), - "p95_latency": float(np.percentile(latencies, 95)), - "p99_latency": float(np.percentile(latencies, 99)), - "mean_latency": float(np.mean(latencies)), - "min_latency": float(np.min(latencies)), - "max_latency": float(np.max(latencies)), - "throughput_req_per_sec": successful_requests / wall_time, - "throughput_tokens_per_sec": total_tokens / wall_time, - } - - -class TestStepAudio2OnlineServing: - """Online serving tests for Step-Audio2.""" - - @pytest.fixture(scope="class") - def server(self): - """Start vLLM-Omni server.""" - with OmniServer(MODEL, ["--stage-configs-path", STAGE_CONFIG]) as omni_server: - yield f"http://{omni_server.host}:{omni_server.port}/v1/chat/completions" - - @pytest.mark.asyncio - async def test_single_request(self, server): - """Test single request works.""" - audio_base64 = create_dummy_audio_base64(duration_sec=2.0) - payload = create_s2st_request(audio_base64) - - async with aiohttp.ClientSession() as session: - latency, tokens, text = await send_request(session, server, payload) - - assert latency > 0 - assert tokens >= 0 - print(f"Single request: latency={latency:.2f}s, tokens={tokens}") - - @pytest.mark.asyncio - @pytest.mark.parametrize("num_concurrent", [1, 2, 4]) - async def test_concurrent_throughput(self, server, num_concurrent): - """Test concurrent throughput at different concurrency levels.""" - audio_base64 = create_dummy_audio_base64(duration_sec=5.0) - payload = create_s2st_request(audio_base64) - - num_requests = num_concurrent * 3 - - stats = await benchmark_concurrent(server, payload, num_concurrent, num_requests) - - assert "error" not in stats - print(f"\nConcurrency={num_concurrent}:") - print(f" p50={stats['p50_latency']:.2f}s, p95={stats['p95_latency']:.2f}s") - print(f" throughput={stats['throughput_req_per_sec']:.2f} req/s") - - -async def run_throughput_benchmark( - server_url: str, - audio_duration: float = 5.0, - concurrency_levels: list[int] = [1, 2, 4, 8], - requests_per_level: int = 10, - warmup: int = 2, -): - """Run full throughput benchmark.""" - audio_base64 = create_dummy_audio_base64(duration_sec=audio_duration) - payload = create_s2st_request(audio_base64) - - print(f"{'=' * 70}") - print("Step-Audio2 Online Serving Throughput Benchmark") - print(f"{'=' * 70}") - print(f"Server: {server_url}") - print(f"Audio duration: {audio_duration}s") - print(f"Requests per level: {requests_per_level}") - print(f"Warmup requests: {warmup}") - print(f"Concurrency levels: {concurrency_levels}") - print(f"{'=' * 70}") - - results = [] - for num_concurrent in concurrency_levels: - print(f"\n[Concurrency={num_concurrent}]") - stats = await benchmark_concurrent(server_url, payload, num_concurrent, requests_per_level, warmup) - results.append(stats) - - if "error" not in stats: - print( - f" Result: p50={stats['p50_latency']:.2f}s, " - f"p95={stats['p95_latency']:.2f}s, " - f"throughput={stats['throughput_req_per_sec']:.2f} req/s, " - f"{stats['throughput_tokens_per_sec']:.1f} tok/s" - ) - - print(f"\n{'=' * 80}") - print("SUMMARY") - print(f"{'=' * 80}") - print( - f"{'Conc':<6} {'Reqs':<6} {'Err':<5} {'Wall(s)':<9} {'p50':<8} {'p95':<8} {'p99':<8} {'Req/s':<8} {'Tok/s':<8}" - ) - print("-" * 80) - - for stats in results: - if "error" in stats: - print(f"{stats.get('num_concurrent', '?'):<6} ERROR: {stats['error']}") - continue - print( - f"{stats['num_concurrent']:<6} " - f"{stats['successful_requests']:<6} " - f"{stats['errors']:<5} " - f"{stats['wall_time']:<9.2f} " - f"{stats['p50_latency']:<8.2f} " - f"{stats['p95_latency']:<8.2f} " - f"{stats['p99_latency']:<8.2f} " - f"{stats['throughput_req_per_sec']:<8.2f} " - f"{stats['throughput_tokens_per_sec']:<8.1f}" - ) - - print("=" * 80) - return results - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description="Step-Audio2 Online Serving Benchmark") - parser.add_argument( - "--server-url", type=str, default="http://localhost:8000/v1/chat/completions", help="vLLM server URL" - ) - parser.add_argument("--audio-duration", type=float, default=5.0, help="Input audio duration in seconds") - parser.add_argument("--concurrency", type=str, default="1,2,4,8", help="Comma-separated concurrency levels to test") - parser.add_argument("--requests-per-level", type=int, default=10, help="Number of requests per concurrency level") - parser.add_argument("--warmup", type=int, default=2, help="Number of warmup requests before each benchmark") - - args = parser.parse_args() - concurrency_levels = [int(x) for x in args.concurrency.split(",")] - - asyncio.run( - run_throughput_benchmark( - args.server_url, - args.audio_duration, - concurrency_levels, - args.requests_per_level, - args.warmup, - ) - ) diff --git a/tests/e2e/online_serving/test_step_audio2_expansion.py b/tests/e2e/online_serving/test_step_audio2_expansion.py new file mode 100644 index 000000000..05beb1131 --- /dev/null +++ b/tests/e2e/online_serving/test_step_audio2_expansion.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E expansion tests for Step-Audio2 online serving (weekly CI). + +Tests speech-to-speech translation via /v1/chat/completions. +Request shape follows examples/online_serving/step_audio2/openai_chat_completion_client.py. +""" + +import os +from pathlib import Path + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_audio +from tests.helpers.runtime import OmniServerParams + +os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + +MODEL = "stepfun-ai/Step-Audio-2-mini" +STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "step_audio2_ci.yaml") +SAMPLE_RATE = 16000 +SEED = 42 +# pyttsx3 synthetic input phrase; Step-Audio2 text output includes codec tokens +# () so we validate audio only, not text↔audio similarity. +SYNTH_PHRASE = "how are you" + +TEST_PARAMS = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=STAGE_CONFIG, + env_dict={"VLLM_IMAGE_FETCH_TIMEOUT": "60"}, + ), + id="step_audio2", + ) +] + +pytestmark = [ + pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True), + pytest.mark.slow, + pytest.mark.tts, +] + + +def _synthetic_audio_base64(duration_sec: int = 2) -> str: + return generate_synthetic_audio( + duration_sec, + 1, + SAMPLE_RATE, + phrase_text=SYNTH_PHRASE, + )["base64"] + + +def _default_sampling_params_list() -> list[dict]: + return [ + { + "temperature": 0.7, + "top_p": 0.9, + "top_k": -1, + "max_tokens": 256, + "seed": SEED, + "detokenize": True, + "repetition_penalty": 1.05, + }, + { + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "max_tokens": 1, + "seed": SEED, + "detokenize": False, + }, + ] + + +def _build_s2st_request_config(omni_server, audio_base64: str) -> dict: + """Build audio-to-audio (S2ST) chat completion request config.""" + audio_url = f"data:audio/wav;base64,{audio_base64}" + return { + "model": omni_server.model, + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are an audio processing assistant. Listen and repeat the audio content.", + } + ], + }, + { + "role": "user", + "content": [ + {"type": "audio_url", "audio_url": {"url": audio_url}}, + {"type": "text", "text": "Please listen to this audio and repeat its content."}, + ], + }, + {"role": "assistant", "content": ""}, + ], + # Thinker text includes undetokenized audio codec placeholders; assert audio only. + "modalities": ["audio"], + "audio_ref_text": SYNTH_PHRASE, + "sampling_params_list": _default_sampling_params_list(), + "extra_body": { + "continue_final_message": True, + "add_generation_prompt": False, + }, + } + + +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_single_s2st_request(omni_server, openai_client) -> None: + """Test single speech-to-speech request via chat completions API.""" + request_config = _build_s2st_request_config( + omni_server, + _synthetic_audio_base64(duration_sec=2), + ) + openai_client.send_omni_request(request_config) + + +@hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.parametrize("request_num", [2, 4]) +def test_concurrent_s2st_requests(omni_server, openai_client, request_num: int) -> None: + """Test concurrent speech-to-speech requests.""" + request_config = _build_s2st_request_config( + omni_server, + _synthetic_audio_base64(duration_sec=2), + ) + responses = openai_client.send_omni_request(request_config, request_num=request_num) + assert len(responses) == request_num diff --git a/tests/e2e/online_serving/test_voxtral_tts.py b/tests/e2e/online_serving/test_voxtral_tts.py deleted file mode 100644 index a86de2100..000000000 --- a/tests/e2e/online_serving/test_voxtral_tts.py +++ /dev/null @@ -1,180 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -E2E Online tests for Voxtral TTS model with text input and audio output. - -These tests verify the /v1/audio/speech endpoint works correctly with -actual model inference, not mocks. -""" - -import os - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - -import pytest - -from tests.helpers.mark import hardware_test -from tests.helpers.runtime import OmniServerParams -from tests.helpers.stage_config import get_deploy_config_path - -MODEL = "mistralai/Voxtral-4B-TTS-2603" - -STAGE_CONFIG = get_deploy_config_path("voxtral_tts.yaml") -EXTRA_ARGS = ["--trust-remote-code", "--enforce-eager", "--disable-log-stats"] -TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, server_args=EXTRA_ARGS)] - - -def verify_wav_audio(content: bytes) -> bool: - """Verify that content is valid WAV audio data.""" - if len(content) < 44: - return False - return content[:4] == b"RIFF" and content[8:12] == b"WAVE" - - -# Minimum expected audio size for a short sentence (~1 second of 24kHz 16-bit mono WAV) -_MIN_AUDIO_BYTES_BASIC = 10000 - - -@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) -class TestVoxtralTTSFixedVoice: - """E2E tests for Voxtral TTS model.""" - - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_speech_english_basic(self, omni_server, openai_client) -> None: - """Test basic English TTS generation.""" - openai_client.send_audio_speech_request( - { - "model": omni_server.model, - "input": "Hello, how are you?", - "voice": "casual_female", - "language": "English", - "response_format": "wav", - "timeout": 120.0, - "min_audio_bytes": _MIN_AUDIO_BYTES_BASIC, - } - ) - - @pytest.mark.core_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_speech_english_streaming(self, omni_server, openai_client) -> None: - """Test basic streaming English TTS generation (PCM via streaming API).""" - openai_client.send_audio_speech_request( - { - "model": omni_server.model, - "input": "Hello, how are you?", - "voice": "casual_female", - "language": "English", - "stream": True, - "response_format": "pcm", - "timeout": 120.0, - "min_audio_bytes": _MIN_AUDIO_BYTES_BASIC, - } - ) - - @pytest.mark.advanced_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_speech_different_voices(self, omni_server, openai_client) -> None: - """Test TTS with different voice presets.""" - voices = ["casual_female", "neutral_male"] - for voice in voices: - openai_client.send_audio_speech_request( - { - "model": omni_server.model, - "input": "Testing voice selection.", - "voice": voice, - "language": "English", - "response_format": "wav", - "timeout": 120.0, - } - ) - - @pytest.mark.advanced_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_speech_speed(self, omni_server, openai_client) -> None: - """Request with speed parameters""" - speeds = [0.5, 1, 1.5, 2, 2.5] - for speed in speeds: - openai_client.send_audio_speech_request( - { - "model": omni_server.model, - "input": "The boy was there when the sun rose.", - "voice": "casual_female", - "language": "English", - "response_format": "wav", - "timeout": 120.0, - "speed": speed, - } - ) - - @pytest.mark.advanced_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_speech_instructions(self, omni_server, openai_client) -> None: - """Request with instructions parameters""" - instructions = [ - "Speak formally", - "Speak angrily", - "Deliver with a sad voice", - "Speak with a chirpy happy voice", - ] - for instruction in instructions: - openai_client.send_audio_speech_request( - { - "model": omni_server.model, - "input": "The boy was there when the sun rose.", - "voice": "casual_female", - "language": "English", - "response_format": "wav", - "timeout": 120.0, - "instructions": instruction, - } - ) - - @pytest.mark.advanced_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_speech_response_formats(self, omni_server, openai_client) -> None: - """Test TTS with different response formats""" - response_formats = ["wav", "mp3"] - for response_format in response_formats: - openai_client.send_audio_speech_request( - { - "model": omni_server.model, - "input": "Testing various response formats.", - "voice": "casual_male", - "language": "English", - "response_format": response_format, - "timeout": 120.0, - } - ) - - @pytest.mark.advanced_model - @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) - def test_speech_batches(self, omni_server, openai_client) -> None: - """Test TTS batches""" - items = [ - {"input": "The birch canoe slid on the smooth planks."}, - {"input": "Glue the sheet to the dark blue background."}, - {"input": "It's easy to tell the depth of a well."}, - {"input": "These days a chicken leg is a rare dish."}, - {"input": "Rice is often served in round bowls."}, - ] - - openai_client.send_audio_speech_batch_http_request( - { - "json": { - "model": omni_server.model, - "items": items, - "voice": "casual_male", - "language": "English", - "response_format": "wav", - }, - "timeout": 120.0, - } - ) diff --git a/tests/e2e/online_serving/test_voxtral_tts_expansion.py b/tests/e2e/online_serving/test_voxtral_tts_expansion.py new file mode 100644 index 000000000..522d18f37 --- /dev/null +++ b/tests/e2e/online_serving/test_voxtral_tts_expansion.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E tests for Voxtral TTS online serving. + +Smoke tests (core_model + advanced_model) and expansion scenarios (slow) +for the /v1/audio/speech endpoint. +""" + +import os + +os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path + +MODEL = "mistralai/Voxtral-4B-TTS-2603" +STAGE_CONFIG = get_deploy_config_path("voxtral_tts.yaml") +EXTRA_ARGS = ["--trust-remote-code", "--enforce-eager", "--disable-log-stats"] +TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, server_args=EXTRA_ARGS)] +_MIN_AUDIO_BYTES_BASIC = 10000 + +pytestmark = [ + pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True), + pytest.mark.slow, + pytest.mark.tts, +] + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_speech_english_basic(omni_server, openai_client) -> None: + """Test basic English TTS generation.""" + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "how are you", + "voice": "casual_female", + "language": "English", + "response_format": "wav", + "timeout": 120.0, + "min_audio_bytes": _MIN_AUDIO_BYTES_BASIC, + } + ) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_speech_english_streaming(omni_server, openai_client) -> None: + """Test basic streaming English TTS generation (PCM via streaming API).""" + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "Hello, how are you?", + "voice": "casual_female", + "language": "English", + "stream": True, + "response_format": "pcm", + "timeout": 120.0, + "min_audio_bytes": _MIN_AUDIO_BYTES_BASIC, + } + ) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_speech_different_voices(omni_server, openai_client) -> None: + """Test TTS with different voice presets.""" + voices = ["casual_female", "neutral_male"] + for voice in voices: + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "Testing voice selection.", + "voice": voice, + "language": "English", + "response_format": "wav", + "timeout": 120.0, + } + ) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_speech_speed(omni_server, openai_client) -> None: + """Request with speed parameters.""" + speeds = [0.5, 1, 1.5, 2, 2.5] + for speed in speeds: + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "The boy was there when the sun rose.", + "voice": "casual_female", + "language": "English", + "response_format": "wav", + "timeout": 120.0, + "speed": speed, + } + ) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_speech_instructions(omni_server, openai_client) -> None: + """Request with instructions parameters.""" + instructions = [ + "Speak formally", + "Speak angrily", + "Deliver with a sad voice", + "Speak with a chirpy happy voice", + ] + for instruction in instructions: + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "The boy was there when the sun rose.", + "voice": "casual_female", + "language": "English", + "response_format": "wav", + "timeout": 120.0, + "instructions": instruction, + } + ) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_speech_response_formats(omni_server, openai_client) -> None: + """Test TTS with different response formats.""" + response_formats = ["wav", "mp3"] + for response_format in response_formats: + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "Testing various response formats.", + "voice": "casual_male", + "language": "English", + "response_format": response_format, + "timeout": 120.0, + } + ) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_speech_batches(omni_server, openai_client) -> None: + """Test TTS batches.""" + items = [ + {"input": "The birch canoe slid on the smooth planks."}, + {"input": "Glue the sheet to the dark blue background."}, + {"input": "It's easy to tell the depth of a well."}, + {"input": "These days a chicken leg is a rare dish."}, + {"input": "Rice is often served in round bowls."}, + ] + + openai_client.send_audio_speech_batch_http_request( + { + "json": { + "model": omni_server.model, + "items": items, + "voice": "casual_male", + "language": "English", + "response_format": "wav", + }, + "timeout": 120.0, + } + ) diff --git a/tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py b/tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py index 32a2dedbd..68acb7c59 100644 --- a/tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py +++ b/tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py @@ -29,7 +29,7 @@ class TestOpenAIAudioListEndpoints: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_list_voices_endpoint(self, omni_server) -> None: """``GET /v1/audio/voices`` returns a non-empty ``voices`` list.""" url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/voices" @@ -45,7 +45,7 @@ class TestOpenAIAudioListEndpoints: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_models_endpoint(self, omni_server) -> None: """``GET /v1/models`` returns at least one model.""" url = f"http://{omni_server.host}:{omni_server.port}/v1/models" diff --git a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py index bbe50a925..15e257959 100644 --- a/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py +++ b/tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py @@ -214,27 +214,6 @@ def _assert_realtime_accuracy(result: dict) -> None: class TestQwen3OmniRealtimeWebSocket: - @pytest.mark.core_model - @pytest.mark.omni - @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) - @pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True) - def test_streaming_audio_input_pcm_output_async_chunk_smoke(self, omni_server) -> None: - """Ready CI: async_chunk on, no send delay, smoke only.""" - pcm16 = _synthetic_pcm16_input() - - result = asyncio.run( - _run_realtime_audio_roundtrip( - omni_server.host, - omni_server.port, - omni_server.model, - pcm16, - chunk_ms=100, - send_delay_ms=0, - ) - ) - - _assert_realtime_smoke(result) - @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) diff --git a/tests/entrypoints/openai_api/test_serving_speech_batch.py b/tests/entrypoints/openai_api/test_serving_speech_batch.py index 484e4cb46..ce0a5e21f 100644 --- a/tests/entrypoints/openai_api/test_serving_speech_batch.py +++ b/tests/entrypoints/openai_api/test_serving_speech_batch.py @@ -125,7 +125,7 @@ class TestSpeechBatchE2E: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_server_params, indirect=True) def test_batch_single_item(self, omni_server) -> None: """Batch with a single item works correctly.""" @@ -147,7 +147,7 @@ class TestSpeechBatchE2E: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_server_params, indirect=True) def test_batch_per_item_voice_override(self, omni_server) -> None: """Per-item voice overrides the batch-level default.""" @@ -175,7 +175,7 @@ class TestSpeechBatchE2E: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_server_params, indirect=True) def test_batch_multiple_languages(self, omni_server) -> None: """Batch items with different languages per item.""" @@ -200,7 +200,7 @@ class TestSpeechBatchE2E: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_server_params, indirect=True) def test_batch_whisper_transcription(self, omni_server) -> None: """Whisper transcription of batch output matches input text.""" @@ -235,7 +235,7 @@ class TestSpeechBatchE2E: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_server_params, indirect=True) def test_batch_result_indices_ordered(self, omni_server) -> None: """Result indices match the input item order.""" @@ -262,7 +262,7 @@ class TestSpeechBatchValidation: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_server_params, indirect=True) def test_batch_empty_items_rejected(self, omni_server) -> None: """Empty items list returns a 4xx error.""" @@ -276,7 +276,7 @@ class TestSpeechBatchValidation: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_server_params, indirect=True) def test_batch_exceeds_max_items(self, omni_server) -> None: """Batch exceeding 32 items returns 400 error.""" @@ -334,7 +334,7 @@ class TestSpeechBatchSize2: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", batch2_server_params, indirect=True) def test_batch2_produces_valid_audio(self, omni_server) -> None: """Batch of 2 items with batched engine produces valid audio.""" @@ -371,7 +371,7 @@ class TestSpeechBatchSize2: @pytest.mark.advanced_model @pytest.mark.tts - @hardware_test(res={"cuda": "H100"}, num_cards=1) + @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", batch2_server_params, indirect=True) def test_batch2_vs_sequential_timing(self, omni_server) -> None: """Compare batch-of-2 vs 2 sequential single requests. diff --git a/tests/entrypoints/test_async_omni.py b/tests/entrypoints/test_async_omni.py index 0d7700e27..7448cc5c0 100644 --- a/tests/entrypoints/test_async_omni.py +++ b/tests/entrypoints/test_async_omni.py @@ -272,69 +272,72 @@ def test_output_kind_is_preserved_with_explicit_sampling_params(output_kind): # End to end tests for ensuring internal manipulation of request ID # in diffusion / Omni models don't leak back to the user. # -# NOTE: It seems like we currently need the shutdowns here, otherwise -# running the tests sequentially seems to leave a zombie process in diffusion -# that can OOM the Omni tests. -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.omni -@pytest.mark.asyncio -@pytest.mark.parametrize( - "req_id", - ["my-req-1", "img_gen-abc123", "chatcmpl-xyz"], - ids=["plain", "prefixed-img", "prefixed-chat"], -) -async def test_diffusion_generate_preserves_request_id(req_id): - """Ensure diffusion model requests don't leak internal UUID-suffixed req id back to user.""" - engine = AsyncOmni(model=DIFFUSION_MODEL) - try: - async for output in engine.generate("a white cat", request_id=req_id): - assert output.request_id == req_id - finally: - engine.shutdown() +# One AsyncOmni per test function (all cases in a single asyncio loop) to avoid +# repeated cold starts. Do not use class/module-scoped engine fixtures here: +# pytest-asyncio uses a function-scoped event loop by default, so reusing an +# engine across tests can hang on the second generate() call. -@hardware_test(res={"cuda": "H100"}, num_cards=1) -@pytest.mark.omni -@pytest.mark.asyncio -@pytest.mark.parametrize( - "req_id", - ["my-req-1", "img_gen-abc123", "chatcmpl-xyz"], - ids=["plain", "prefixed-img", "prefixed-chat"], -) -async def test_omni_generate_preserves_request_id(req_id): - """Ensure omni model requests don't leak internal UUID-suffixed req id back to user.""" - engine = AsyncOmni(model=OMNI_MODEL, stage_configs_path=OMNI_STAGE_CONFIG) - try: - async for output in engine.generate( - "Say hello in one word.", - request_id=req_id, - output_modalities=["text"], - ): - assert output.request_id == req_id - finally: - engine.shutdown() +# Covers: +# * plain client ids (``my-req-1``) +# * OpenAI-style prefixed ids (``img_gen-*``, ``chatcmpl-*``) that AsyncOmni +# suffixes internally for engine routing — streamed outputs must still echo +# the caller-visible id, not the internal UUID-suffixed id +# * empty ``request_id`` — server assigns a non-empty id for the caller +_DIFFUSION_REQ_IDS = ["my-req-1", "img_gen-abc123", "chatcmpl-xyz"] +_OMNI_REQ_IDS = ["my-req-1", "img_gen-abc123", "chatcmpl-xyz"] @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.omni @pytest.mark.asyncio -async def test_diffusion_generate_empty_request_id(): - """Empty request_id should get a generated internal ID, not stay empty (diffusion)""" +async def test_diffusion_generate_request_id(): + """Diffusion E2E request-id contract (``riverclouds/qwen_image_random``). + + Scenarios (one engine, sequential ``generate`` calls): + - plain id ``my-req-1`` + - image-style prefix ``img_gen-abc123`` + - chat-style prefix ``chatcmpl-xyz`` + - empty ``request_id`` → output id is non-empty (auto-assigned) + + Each streaming output must expose the user-supplied id unchanged; internal + UUID suffixing must not leak into ``output.request_id``. + """ engine = AsyncOmni(model=DIFFUSION_MODEL) try: + for req_id in _DIFFUSION_REQ_IDS: + async for output in engine.generate("a white cat", request_id=req_id): + assert output.request_id == req_id async for output in engine.generate("a white cat", request_id=""): assert output.request_id != "" finally: engine.shutdown() -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.omni @pytest.mark.asyncio -async def test_omni_generate_empty_request_id(): - """Empty request_id should get a generated internal ID, not stay empty (omni)""" +async def test_omni_generate_request_id(): + """Omni E2E request-id contract (``Qwen/Qwen2.5-Omni-7B``, thinker-only stage). + + Same scenarios as ``test_diffusion_generate_request_id``: + - plain id ``my-req-1`` + - image-style prefix ``img_gen-abc123`` + - chat-style prefix ``chatcmpl-xyz`` + - empty ``request_id`` → output id is non-empty (auto-assigned) + + Text modality only; asserts caller-visible ids are preserved across the + multi-stage orchestrator path on H100. + """ engine = AsyncOmni(model=OMNI_MODEL, stage_configs_path=OMNI_STAGE_CONFIG) try: + for req_id in _OMNI_REQ_IDS: + async for output in engine.generate( + "Say hello in one word.", + request_id=req_id, + output_modalities=["text"], + ): + assert output.request_id == req_id async for output in engine.generate( "Say hello in one word.", request_id="", diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index de77722b3..ef7cb5f13 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -17,6 +17,7 @@ import soundfile as sf from PIL import Image from tests.helpers.media import ( + convert_audio_bytes_to_text, cosine_similarity_text, ) @@ -374,8 +375,15 @@ def _estimate_voice_gender_from_audio(audio_bytes: bytes) -> str: return "unknown" -def _assert_preset_voice_gender_from_audio(audio_bytes: bytes | None, voice_name: str | None) -> None: +def _assert_preset_voice_gender_from_audio( + audio_bytes: bytes | None, + voice_name: str | None, + *, + response_format: str | None = None, +) -> None: """If ``voice_name`` matches a known preset, assert classifier gender matches (skip when unknown).""" + if response_format == "pcm": + return if not voice_name or not audio_bytes: return key = str(voice_name).lower() @@ -432,6 +440,61 @@ def _assert_pcm_int16_speech_hnr(audio_bytes: bytes, min_hnr_db: float = _MIN_PC ) +def _response_has_audio_output(response: Any) -> bool: + if response.audio_bytes: + return len(response.audio_bytes) > 0 + if isinstance(getattr(response, "audio_content", None), str) and response.audio_content.strip(): + return True + audio_data = getattr(response, "audio_data", None) + return bool(audio_data) + + +def _omni_assertion_needs_audio_transcript(request_config: dict[str, Any], run_level: str) -> bool: + if run_level not in {"advanced_model", "full_model"}: + return False + modalities = request_config.get("modalities", ["text", "audio"]) + if "audio" not in modalities: + return False + keywords_dict = request_config.get("key_words", {}) or {} + if keywords_dict.get("audio") and "text" not in modalities: + return True + if request_config.get("audio_ref_text"): + return True + return "text" in modalities + + +def _speech_assertion_needs_audio_transcript(request_config: dict[str, Any], run_level: str) -> bool: + if run_level not in {"advanced_model", "full_model"}: + return False + if request_config.get("response_format") == "pcm": + return False + return bool(request_config.get("input")) + + +def _resolve_audio_transcript( + response: Any, + request_config: dict[str, Any], + run_level: str, + *, + speech_api: bool, +) -> str | None: + """Run Whisper only when this run_level / request_config needs a transcript for assertions.""" + needs = ( + _speech_assertion_needs_audio_transcript(request_config, run_level) + if speech_api + else _omni_assertion_needs_audio_transcript(request_config, run_level) + ) + if not needs: + return None + existing = getattr(response, "audio_content", None) + if isinstance(existing, str) and existing.strip(): + return existing + audio_bytes = getattr(response, "audio_bytes", None) + if not audio_bytes: + return None + return convert_audio_bytes_to_text(audio_bytes) + + def assert_omni_response(response: Any, request_config: dict[str, Any], run_level): """ Validate response results. @@ -447,15 +510,18 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve modalities = request_config.get("modalities", ["text", "audio"]) if run_level in {"advanced_model", "full_model"}: + transcript = _resolve_audio_transcript(response, request_config, run_level, speech_api=False) # Verify output success if "audio" in modalities: - assert response.audio_content is not None, "No audio output is generated" - print(f"audio content is: {response.audio_content}") + assert _response_has_audio_output(response), "No audio output is generated" + if transcript is not None: + print(f"audio content is: {transcript}") speaker = request_config.get("speaker") if speaker: _assert_preset_voice_gender_from_audio( response.audio_bytes, speaker, + response_format=request_config.get("response_format"), ) if "text" in modalities: assert response.text_content is not None, "No text output is generated" @@ -474,7 +540,8 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve ) else: if keywords: - audio_lower = response.audio_content.lower() + assert transcript is not None, "No audio transcript for keyword validation" + audio_lower = transcript.lower() assert any(str(kw).lower() in audio_lower for kw in keywords), ( "The output does not contain any of the keywords." ) @@ -484,7 +551,7 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve audio_ref_text = request_config.get("audio_ref_text") similarity_threshold = request_config.get("similarity_threshold", 0.8) if "text" in modalities: - transcript = (response.audio_content or "").strip() + assert transcript is not None, "No audio transcript for similarity validation" text_output = (response.text_content or "").strip() # For very short outputs (e.g. one-word answers), n-gram cosine # similarity with length penalty is unreliable because Whisper @@ -513,8 +580,9 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve print(f"similarity is: {similarity}") assert similarity > similarity_threshold, "The audio content is not same as the text" if audio_ref_text: + assert transcript is not None, "No audio transcript for reference-text validation" audio_similarity = cosine_similarity_text( - response.audio_content.lower(), + transcript.strip().lower(), str(audio_ref_text).lower(), ) assert audio_similarity > similarity_threshold, ( @@ -557,8 +625,6 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any], req_fmt = request_config.get("response_format") if req_fmt == "pcm" and response.audio_bytes: - min_hnr_db = float(request_config.get("min_hnr_db", _MIN_PCM_SPEECH_HNR_DB)) - _assert_pcm_int16_speech_hnr(response.audio_bytes, min_hnr_db=min_hnr_db) if response.audio_format: assert "pcm" in response.audio_format.lower(), ( f"Expected audio/pcm content-type, got {response.audio_format!r}" @@ -566,18 +632,27 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any], elif req_fmt == "wav" and response.audio_format: assert req_fmt in response.audio_format - if run_level in {"advanced_model", "full_model"} and req_fmt != "pcm": - expected_text = request_config.get("input") - if expected_text: - transcript = (response.audio_content or "").strip() - print(f"audio content is: {transcript}") - print(f"input text is: {expected_text}") - similarity = cosine_similarity_text(transcript.lower(), expected_text.lower()) - print(f"Cosine similarity: {similarity:.3f}") - assert similarity > 0.9, ( - f"Transcript doesn't match input: similarity={similarity:.2f}, transcript='{transcript}'" - ) - _assert_preset_voice_gender_from_audio(response.audio_bytes, request_config.get("voice")) + if run_level in {"advanced_model", "full_model"}: + if req_fmt == "pcm" and response.audio_bytes: + min_hnr_db = float(request_config.get("min_hnr_db", _MIN_PCM_SPEECH_HNR_DB)) + _assert_pcm_int16_speech_hnr(response.audio_bytes, min_hnr_db=min_hnr_db) + + transcript = _resolve_audio_transcript(response, request_config, run_level, speech_api=True) + if transcript is not None: + expected_text = request_config.get("input") + if expected_text: + print(f"audio content is: {transcript}") + print(f"input text is: {expected_text}") + similarity = cosine_similarity_text(transcript.strip().lower(), str(expected_text).lower()) + print(f"Cosine similarity: {similarity:.3f}") + assert similarity > 0.9, ( + f"Transcript doesn't match input: similarity={similarity:.2f}, transcript='{transcript}'" + ) + _assert_preset_voice_gender_from_audio( + response.audio_bytes, + request_config.get("voice"), + response_format=request_config.get("response_format"), + ) def assert_diffusion_response(response: "DiffusionResponse", request_config: dict[str, Any], run_level: str = None): diff --git a/tests/helpers/env.py b/tests/helpers/env.py index 6dee5e610..dcf2484cd 100644 --- a/tests/helpers/env.py +++ b/tests/helpers/env.py @@ -201,6 +201,7 @@ def run_pre_test_cleanup() -> None: wait_for_gpu_memory_to_clear( devices=list(range(num_gpus)), threshold_ratio=0.05, + timeout_s=60, ) except Exception as e: print(f"Pre-test cleanup note: {e}") diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 8fd8cf9d5..89ed91e5f 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -45,7 +45,6 @@ from tests.helpers.assertions import ( from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.media import ( _merge_base64_audio_to_segment, - convert_audio_bytes_to_text, decode_b64_image, ) from vllm_omni.config.stage_config import resolve_deploy_yaml @@ -898,16 +897,13 @@ class OpenAIClientHandler: if details := getattr(chunk.usage, "prompt_tokens_details", None): result.cached_tokens = details.cached_tokens - audio_content = None if audio_data: merged_seg = _merge_base64_audio_to_segment(audio_data) wav_buf = BytesIO() merged_seg.export(wav_buf, format="wav") result.audio_bytes = wav_buf.getvalue() - audio_content = convert_audio_bytes_to_text(result.audio_bytes) result.text_content = text_content result.audio_data = audio_data - result.audio_content = audio_content result.e2e_latency = time.perf_counter() - wall_start result.success = True except Exception as e: @@ -932,12 +928,9 @@ class OpenAIClientHandler: result.prompt_tokens = usage.prompt_tokens if details := getattr(usage, "prompt_tokens_details", None): result.cached_tokens = details.cached_tokens - audio_content = None if audio_data: result.audio_bytes = base64.b64decode(audio_data) - audio_content = convert_audio_bytes_to_text(result.audio_bytes) result.text_content = text_content - result.audio_content = audio_content result.e2e_latency = time.perf_counter() - wall_start if chat_completion.choices and chat_completion.choices[0].logprobs is not None: result.logprobs = chat_completion.choices[0].logprobs.content @@ -1587,6 +1580,8 @@ class OpenAIClientHandler: extra_body["mm_processor_kwargs"] = mm if "sampling_params_list" in request_config: extra_body["sampling_params_list"] = request_config["sampling_params_list"] + if request_config.get("extra_body"): + extra_body.update(request_config["extra_body"]) create_kwargs: dict[str, Any] = { "model": request_config.get("model"), @@ -1648,8 +1643,8 @@ class OpenAIClientHandler: Process streaming /v1/audio/speech responses into an OmniResponse. This mirrors _process_stream_omni_response but operates on low-level - audio bytes and produces an OmniResponse with audio_content filled - from Whisper transcription. + audio bytes. Whisper transcription runs in assert_audio_speech_response + when the run_level requires it. """ result = OmniResponse() @@ -1685,14 +1680,9 @@ class OpenAIClientHandler: raise TypeError(f"Unsupported audio speech streaming response type: {type(response)}") raw_bytes = bytes(data) - if response_format == "pcm": - transcript = None - else: - transcript = convert_audio_bytes_to_text(raw_bytes) # Populate OmniResponse. result.audio_bytes = raw_bytes - result.audio_content = transcript result.e2e_latency = time.perf_counter() - wall_start result.success = True result.audio_format = getattr(response, "response", None) @@ -1725,13 +1715,7 @@ class OpenAIClientHandler: else: raise TypeError(f"Unsupported audio speech response type: {type(response)}") - if response_format == "pcm": - transcript = None - else: - transcript = convert_audio_bytes_to_text(raw_bytes) - result.audio_bytes = raw_bytes - result.audio_content = transcript result.e2e_latency = time.perf_counter() - wall_start result.success = True result.audio_format = getattr(response, "response", None) @@ -2796,25 +2780,6 @@ class OmniRunnerHandler: # --------------------------------------------------------------------------- -def _core_model_stage_config_path_with_dummy_load_format(stage_config_path: str | None, run_level: str) -> str | None: - """For ``core_model`` runs, patch every stage in the deploy YAML to ``load_format: dummy``.""" - if run_level != "core_model" or stage_config_path is None: - return stage_config_path - from tests.helpers.stage_config import modify_stage_config - - with open(stage_config_path, encoding="utf-8") as f: - cfg = yaml.safe_load(f) or {} - new_schema_stages = cfg.get("stages") - stage_key = "stages" if new_schema_stages is not None else "stage_args" - update_path = "load_format" if new_schema_stages is not None else "engine_args.load_format" - stage_entries = cfg.get(stage_key, []) - stage_ids = [stage["stage_id"] for stage in stage_entries if "stage_id" in stage] - return modify_stage_config( - stage_config_path, - updates={stage_key: {stage_id: {update_path: "dummy"} for stage_id in stage_ids}}, - ) - - def iter_omni_server( request: Any, run_level: str, @@ -2822,25 +2787,13 @@ def iter_omni_server( omni_fixture_lock: threading.Lock, ) -> Generator[Any, Any, None]: """Start/stop an Omni HTTP server; used by ``omni_server`` / ``omni_server_function`` fixtures.""" - from tests.helpers.stage_config import modify_stage_config + from tests.helpers.stage_config import stage_config_path_for_run_level with omni_fixture_lock: params: OmniServerParams = request.param model = model_prefix + params.model port = params.port - stage_config_path = params.stage_config_path - if run_level in {"advanced_model", "full_model"} and stage_config_path is not None: - with open(stage_config_path, encoding="utf-8") as f: - cfg = yaml.safe_load(f) or {} - new_schema_stages = cfg.get("stages") - stage_key = "stages" if new_schema_stages is not None else "stage_args" - delete_path = "load_format" if new_schema_stages is not None else "engine_args.load_format" - stage_entries = cfg.get(stage_key, []) - stage_ids = [stage["stage_id"] for stage in stage_entries if "stage_id" in stage] - stage_config_path = modify_stage_config( - stage_config_path, - deletes={stage_key: {stage_id: [delete_path] for stage_id in stage_ids}}, - ) + stage_config_path = stage_config_path_for_run_level(params.stage_config_path, run_level) server_args = params.server_args or [] if params.use_omni and params.stage_init_timeout is not None: @@ -2905,6 +2858,8 @@ def iter_omni_runner( omni_fixture_lock: threading.Lock, ) -> Generator[Any, None, None]: """Yield an :class:`OmniRunner`; used by ``omni_runner`` / ``omni_runner_function`` fixtures.""" + from tests.helpers.stage_config import stage_config_path_for_run_level + with omni_fixture_lock: param = request.param if not isinstance(param, (tuple, list)) or len(param) not in (2, 3): @@ -2918,7 +2873,7 @@ def iter_omni_runner( else: model, stage_config_path, extra = param[0], param[1], param[2] extra_omni_kwargs = dict(extra) if extra is not None else {} - stage_config_path = _core_model_stage_config_path_with_dummy_load_format(stage_config_path, run_level) + stage_config_path = stage_config_path_for_run_level(stage_config_path, run_level) model = model_prefix + model with OmniRunner(model, seed=42, stage_configs_path=stage_config_path, **extra_omni_kwargs) as runner: print("OmniRunner started successfully") diff --git a/tests/helpers/stage_config.py b/tests/helpers/stage_config.py index e34680be0..62ae38558 100644 --- a/tests/helpers/stage_config.py +++ b/tests/helpers/stage_config.py @@ -627,7 +627,80 @@ def get_deploy_config_path(rel_path: str) -> str: return str(_DEPLOY_DIR / rel_path) +def _get_config_value_by_path(config_dict: dict, path: str) -> Any: + """Read a dot-separated path from a nested dict (e.g. ``engine_args.load_format``).""" + current: Any = config_dict + for key in path.split("."): + if not isinstance(current, dict) or key not in current: + return None + current = current[key] + return current + + +def _stage_load_format_paths(stage_config_path: str) -> tuple[str, str, list[int]]: + """Return ``(stage_key, load_format_field_path, stage_ids)`` for a deploy YAML.""" + with open(stage_config_path, encoding="utf-8") as f: + cfg = yaml.safe_load(f) or {} + new_schema_stages = cfg.get("stages") + stage_key = "stages" if new_schema_stages is not None else "stage_args" + load_format_path = "load_format" if new_schema_stages is not None else "engine_args.load_format" + stage_entries = cfg.get(stage_key, []) + stage_ids = [stage["stage_id"] for stage in stage_entries if "stage_id" in stage] + return stage_key, load_format_path, stage_ids + + +def _add_dummy_load_format( + stage_config_path: str | None, + run_level: str, +) -> str | None: + """For ``core_model`` runs, patch every stage in the deploy YAML to ``load_format: dummy``.""" + if run_level != "core_model" or stage_config_path is None: + return stage_config_path + stage_key, load_format_path, stage_ids = _stage_load_format_paths(stage_config_path) + return modify_stage_config( + stage_config_path, + updates={stage_key: {stage_id: {load_format_path: "dummy"} for stage_id in stage_ids}}, + ) + + +def _delete_dummy_load_format( + stage_config_path: str | None, + run_level: str, +) -> str | None: + """For ``advanced_model`` / ``full_model``, strip ``load_format: dummy`` so real weights load.""" + if run_level not in {"advanced_model", "full_model"} or stage_config_path is None: + return stage_config_path + stage_key, load_format_path, _stage_ids = _stage_load_format_paths(stage_config_path) + with open(stage_config_path, encoding="utf-8") as f: + cfg = yaml.safe_load(f) or {} + stage_entries = cfg.get(stage_key, []) + + deletes: dict[int, list[str]] = {} + for stage in stage_entries: + stage_id = stage.get("stage_id") + if stage_id is None: + continue + if _get_config_value_by_path(stage, load_format_path) == "dummy": + deletes[stage_id] = [load_format_path] + + if not deletes: + return stage_config_path + + return modify_stage_config( + stage_config_path, + deletes={stage_key: deletes}, + ) + + +def stage_config_path_for_run_level(stage_config_path: str | None, run_level: str) -> str | None: + """Apply run-level deploy YAML tweaks for weight loading (dummy vs real).""" + if run_level in {"advanced_model", "full_model"}: + return _delete_dummy_load_format(stage_config_path, run_level) + return _add_dummy_load_format(stage_config_path, run_level) + + __all__ = [ - "modify_stage_config", "get_deploy_config_path", + "modify_stage_config", + "stage_config_path_for_run_level", ] diff --git a/tests/model_executor/models/step_audio2/test_step_audio2_thinker.py b/tests/model_executor/models/step_audio2/test_step_audio2_thinker.py new file mode 100644 index 000000000..8ed1ce9a4 --- /dev/null +++ b/tests/model_executor/models/step_audio2/test_step_audio2_thinker.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for Step-Audio2 thinker processor, encoder, and token handling.""" + +import numpy as np +import pytest +import torch + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def create_dummy_audio(sample_rate: int = 16000, duration_sec: float = 1.0) -> tuple[np.ndarray, int]: + """Create a dummy audio signal (sine wave) for testing.""" + t = np.linspace(0, duration_sec, int(sample_rate * duration_sec), dtype=np.float32) + audio = 0.5 * np.sin(2 * np.pi * 440 * t) + return audio, sample_rate + + +def test_processor_loads_correctly(): + """Test that Step-Audio2 processor can be imported and initialized.""" + from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( + StepAudio2MultiModalProcessor, + StepAudio2ProcessingInfo, + StepAudio2Processor, + ) + + assert StepAudio2MultiModalProcessor is not None + assert StepAudio2ProcessingInfo is not None + assert StepAudio2Processor is not None + + +def test_audio_preprocessing(): + """Test that audio preprocessing produces correct tensor shapes.""" + from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( + log_mel_spectrogram, + padding_mels, + ) + + audio, _sample_rate = create_dummy_audio(sample_rate=16000, duration_sec=1.0) + + mel = log_mel_spectrogram(audio) + + assert mel.ndim == 2, f"Expected 2D tensor, got {mel.ndim}D" + assert mel.shape[0] == 128, f"Expected 128 mel bins, got {mel.shape[0]}" + + mels = [mel, mel] + padded_mels, _mel_lens = padding_mels(mels) + + assert padded_mels.ndim == 3, f"Expected 3D tensor, got {padded_mels.ndim}D" + assert padded_mels.shape[0] == 2, f"Expected batch size 2, got {padded_mels.shape[0]}" + assert padded_mels.shape[1] == 128, f"Expected 128 mel bins, got {padded_mels.shape[1]}" + + +def test_feature_length_calculation(): + """Test that audio feature length calculation is correct.""" + from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( + calculate_audio_feature_length, + ) + + assert calculate_audio_feature_length(1000) == 125 + + assert calculate_audio_feature_length(100) > 0 + + assert calculate_audio_feature_length(1) >= 1 + + +def test_token_config_constants(): + """Test that token configuration constants are set correctly.""" + from vllm_omni.model_executor.models.step_audio2.step_audio2_constants import ( + DEFAULT_TOKEN_CONFIG, + STEP_AUDIO2_AUDIO_END, + STEP_AUDIO2_AUDIO_PATCH_TOKEN_ID, + STEP_AUDIO2_AUDIO_START, + STEP_AUDIO2_AUDIO_VOCAB_SIZE, + STEP_AUDIO2_TEXT_MAX, + ) + + assert STEP_AUDIO2_TEXT_MAX < STEP_AUDIO2_AUDIO_START, "Text tokens should come before audio tokens" + assert STEP_AUDIO2_AUDIO_END == STEP_AUDIO2_AUDIO_START + STEP_AUDIO2_AUDIO_VOCAB_SIZE - 1 + assert STEP_AUDIO2_AUDIO_PATCH_TOKEN_ID > STEP_AUDIO2_TEXT_MAX + assert STEP_AUDIO2_AUDIO_PATCH_TOKEN_ID < STEP_AUDIO2_AUDIO_START + + assert DEFAULT_TOKEN_CONFIG.text_max == STEP_AUDIO2_TEXT_MAX + assert DEFAULT_TOKEN_CONFIG.audio_start == STEP_AUDIO2_AUDIO_START + + +def test_mm_field_config_structure(): + """Test that multimodal field config is properly structured for vLLM batching.""" + from vllm.multimodal.inputs import MultiModalFieldConfig + + audio_lens = torch.tensor([100, 200, 150], dtype=torch.int32) + + field_config = MultiModalFieldConfig.flat_from_sizes("audio", audio_lens) + + assert field_config is not None + + +def test_audio_encoder_output_shape(): + """Test that audio encoder produces correct output shapes.""" + from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( + Adaptor, + AudioEncoder, + ) + + encoder = AudioEncoder(n_mels=128, n_ctx=1500, n_state=512, n_head=8, n_layer=6) + + batch_size = 2 + n_mels = 128 + time_steps = 400 + x = torch.randn(batch_size, n_mels, time_steps) + x_len = torch.tensor([time_steps, time_steps // 2], dtype=torch.int32) + + encoded, _encoded_lens = encoder(x, x_len) + + assert encoded.ndim == 3, f"Expected 3D tensor, got {encoded.ndim}D" + assert encoded.shape[0] == batch_size + assert encoded.shape[2] == 512 + + adapter = Adaptor(n_state=512, n_hidden=4096, kernel_size=3, stride=2) + adapted = adapter(encoded) + + assert adapted.ndim == 3 + assert adapted.shape[0] == batch_size + assert adapted.shape[2] == 4096 + + +def test_token_separation(): + """Test that token separation works correctly.""" + from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( + StepAudio2ThinkerForConditionalGeneration, + ) + + token_ids = [100, 200, 151700, 300, 151800, 400] + + text_tokens, audio_tokens = StepAudio2ThinkerForConditionalGeneration.separate_tokens(token_ids) + + assert text_tokens == [100, 200, 300, 400] + assert audio_tokens == [4, 104] + + +def test_has_audio_output(): + """Test detection of audio tokens in output.""" + from vllm_omni.model_executor.models.step_audio2.step_audio2_thinker import ( + StepAudio2ThinkerForConditionalGeneration, + ) + + text_only = [100, 200, 300] + assert not StepAudio2ThinkerForConditionalGeneration.has_audio_output(text_only) + + with_audio = [100, 200, 151700, 300] + assert StepAudio2ThinkerForConditionalGeneration.has_audio_output(with_audio)