diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 46ed531f3..22da9617b 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -123,6 +123,48 @@ steps: - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: h100_2 + - label: "Omni · MiniCPM-o 4.5 Test" + source_file_dependencies: + - tests/e2e/offline_inference/test_minicpmo_4_5.py + - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/deploy/minicpmo_4_5.yaml + timeout_in_minutes: 50 + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model' + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: "TTS · VoxCPM2 Test" source_file_dependencies: - tests/e2e/offline_inference/test_voxcpm2_tts.py diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 90e39b12a..664c6c711 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -134,6 +134,82 @@ steps: " mirror_hardwares: h100_2 + - label: "Omni · MiniCPM-o 4.5 Offline Test" + source_file_dependencies: + - tests/e2e/offline_inference/test_minicpmo_4_5.py + - vllm_omni/deploy/minicpmo_4_5.yaml + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + " + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + - label: "Omni · MiniCPM-o 4.5 Online Test" + source_file_dependencies: + - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/deploy/minicpmo_4_5.yaml + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + " + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: "TTS · VoxCPM2 Test" source_file_dependencies: - tests/e2e/online_serving/test_voxcpm2_tts.py diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 4050cca8d..e05178c52 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -34,3 +34,25 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT + + - group: ":card_index_dividers: Omni Model Test" + key: nightly-omni-group + depends_on: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" + steps: + - label: ":robot: Omni · MiniCPM-o 4.5 Test" + agents: + queue: "ascend-a3" + resource_class: "npu-2" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: + - kubernetes: + podSpecPatch: + imagePullSecrets: + - name: swr-secret + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model' diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index 317517398..956e8acd6 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -1,4 +1,25 @@ steps: + - group: ":card_index_dividers: Omni Model Test" + key: ready-omni-group + depends_on: upload-ready-pipeline + steps: + - label: ":robot: Omni · MiniCPM-o 4.5 Test" + agents: + queue: "ascend-a3" + resource_class: "npu-2" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: + - kubernetes: + podSpecPatch: + imagePullSecrets: + - name: swr-secret + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + - group: ":card_index_dividers: TTS Model Test" key: ready-tts-group depends_on: upload-ready-pipeline diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index a3cede3a7..c0b4af9b6 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -37,6 +37,9 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pyttsx3>=2.99" \ "opencv-python>=4.12.0.88" \ "imageio[ffmpeg]>=0.6.0" \ + "hyperpyyaml>=1.2.0" \ + "onnx>=1.16.0" \ + "librosa>=0.10.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 && \ @@ -45,6 +48,17 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 +# MiniCPM-o 4.5 Stage-2 vocoder assets: s3tokenizer plus flashcosyvoice, which +# is shipped by step-audio2. --no-deps keeps pip from replacing the Ascend torch +# build and from shadowing onnxruntime-cann with the vanilla onnxruntime wheel; +# the deps these two actually need are installed above. +RUN python3 -m pip install -v --no-deps \ + "s3tokenizer>=0.3.0" \ + "step-audio2>=1.0.0" \ + --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ + --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ + --default-timeout=60 + ENV VLLM_WORKER_MULTIPROC_METHOD=spawn ENTRYPOINT [] diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index 1369d8c85..b86d7c0f8 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -37,6 +37,9 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pyttsx3>=2.99" \ "opencv-python>=4.12.0.88" \ "imageio[ffmpeg]>=0.6.0" \ + "hyperpyyaml>=1.2.0" \ + "onnx>=1.16.0" \ + "librosa>=0.10.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 && \ @@ -45,6 +48,17 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 +# MiniCPM-o 4.5 Stage-2 vocoder assets: s3tokenizer plus flashcosyvoice, which +# is shipped by step-audio2. --no-deps keeps pip from replacing the Ascend torch +# build and from shadowing onnxruntime-cann with the vanilla onnxruntime wheel; +# the deps these two actually need are installed above. +RUN python3 -m pip install -v --no-deps \ + "s3tokenizer>=0.3.0" \ + "step-audio2>=1.0.0" \ + --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ + --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ + --default-timeout=60 + ENV VLLM_WORKER_MULTIPROC_METHOD=spawn ENTRYPOINT [] diff --git a/examples/offline_inference/minicpmo/README.md b/examples/offline_inference/minicpmo/README.md index bb6ce2568..b80fc0aa6 100644 --- a/examples/offline_inference/minicpmo/README.md +++ b/examples/offline_inference/minicpmo/README.md @@ -6,7 +6,7 @@ Two-stage pipeline: **thinker** (multimodal understanding) → **talker + Token2 ## Setup -- Install talker deps: `pip install 'vllm-omni[minicpmo]'` (or `stepaudio2-minicpmo`) +- Install talker deps: `pip install stepaudio2-minicpmo` - `--trust-remote-code` is always passed by `end2end.py` (required for MiniCPMO) - See [stage configuration docs](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory tuning diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 39bbef09b..148fbb5e2 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -13,10 +13,7 @@ capability boundary, and validation scope, see Install vLLM-Omni with the MiniCPM-o talker dependencies: ```bash -pip install 'vllm-omni[minicpmo]' - -# From a source checkout: -pip install -e '.[minicpmo]' +pip install stepaudio2-minicpmo ``` The `minicpmo` extra installs `stepaudio2-minicpmo` and its audio dependencies, diff --git a/pyproject.toml b/pyproject.toml index c71e17b4e..c442c3164 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ dev = [ "kernels==0.14.1", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0) "s3tokenizer==0.3.0", # for tts test "step-audio2==1.0.0", # Step-Audio2 Token2Wav (flashcosyvoice / hyperpyyaml) + "stepaudio2-minicpmo", # for MiniCPM-o 4.5 talker vocoder (stepaudio2-minicpmo Token2wav) ] demo = [ @@ -88,18 +89,6 @@ local = [ "FlagEmbedding", ] -# Optional runtime dependencies for the MiniCPM-o 4.5 talker pipeline. -# The talker loads MiniCPMTTS via `trust_remote_code` from the model repo, -# and that module performs `from stepaudio2 import Token2wav` against the -# MiniCPM-o-flavored Token2wav vocoder (PyPI: `stepaudio2-minicpmo`, NOT -# upstream `stepfun-ai/Step-Audio2` — the upstream signature rejects -# `n_timesteps` and crashes at construction). A clean `vllm-omni` install -# does not pull this in by default to avoid forcing the dep onto users -# who only need other models; opt in with `pip install vllm-omni[minicpmo]`. -minicpmo = [ - "stepaudio2-minicpmo", -] - # Optional Blackwell-only fused-bias FP8 GEMM; package is `quack-kernels` (imported # as `quack`), not the unrelated `quack`. See docs/user_guide/quantization/fp8.md. quack = [ diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 73d0fd5a5..dddc9fac4 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -256,11 +256,10 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ rejects `n_timesteps`). Install via the published extra: ```bash - pip install 'vllm-omni[minicpmo]' + pip install stepaudio2-minicpmo ``` - Equivalent direct install: `pip install stepaudio2-minicpmo`. A - missing dep raises `ImportError` at first request with the same + A missing dep raises `ImportError` at first request with the same install hint instead of silently emitting empty audio. - **TTS conditioning**: the MiniCPM stage bridge can condition speech from diff --git a/tests/core/sched/test_omni_ar_scheduler_streaming.py b/tests/core/sched/test_omni_ar_scheduler_streaming.py index 9c0533059..06037d4cf 100644 --- a/tests/core/sched/test_omni_ar_scheduler_streaming.py +++ b/tests/core/sched/test_omni_ar_scheduler_streaming.py @@ -137,6 +137,56 @@ def test_resumable_session_terminal_is_not_marked_as_segment_boundary() -> None: assert output.is_segment_finished is False +def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> None: + """A decode step that neither stops nor carries an inter-stage payload. + + ``finished`` is only assigned when the request stops, yet the async-chunk + save condition reads it for every request, so this step used to raise + ``UnboundLocalError: cannot access local variable 'finished'``. + """ + session = _make_request() + session.status = RequestStatus.RUNNING + + sched = MagicMock() + sched.requests = {session.request_id: session} + sched.perf_metrics = None + sched.structured_output_manager.should_advance.return_value = False + sched._update_request_with_output.return_value = ([42], False) + sched._process_kv_transfer_trigger.return_value = False + sched.chunk_transfer_adapter = MagicMock() + sched.running = [session] + sched.waiting_for_transfer_free = set() + sched.transfer_triggered_requests = set() + sched.active_kv_transfers = set() + sched.pending_stop_after_extraction = set() + sched.connector = None + sched.kv_cache_manager.take_events.return_value = None + sched.finished_req_ids_dict = {} + sched.make_stats.return_value = None + + scheduler_output = MagicMock(spec=SchedulerOutput) + scheduler_output.num_scheduled_tokens = {session.request_id: 1} + scheduler_output.scheduled_spec_decode_tokens = {} + scheduler_output.num_invalid_spec_tokens = 0 + + model_runner_output = MagicMock(spec=ModelRunnerOutput) + model_runner_output.sampled_token_ids = [[42]] + model_runner_output.logprobs = None + model_runner_output.prompt_logprobs_dict = {} + model_runner_output.pooler_output = None + model_runner_output.num_nans_in_logits = None + model_runner_output.kv_connector_output = None + model_runner_output.cudagraph_stats = None + model_runner_output.req_id_to_index = {session.request_id: 0} + model_runner_output.routed_experts = None + model_runner_output.inter_stage_outputs = None + + OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output) + + # Nothing to hand downstream: no payload, no segment boundary, not finished. + sched.chunk_transfer_adapter.save_async.assert_not_called() + + def test_stage0_streaming_update_discards_outstanding_async_placeholder_token() -> None: sched = _make_scheduler(stage_id=0) session = _make_request() diff --git a/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py new file mode 100644 index 000000000..c71dc2d16 --- /dev/null +++ b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Invalid audio format rejection on MiniCPM-o 4.5: ``POST /v1/chat/completions``.""" + +import os + +os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +from tests.helpers.stage_config import get_deploy_config_path + +pytestmark = [pytest.mark.slow, pytest.mark.omni, pytest.mark.full_model] + +_MINICPMO_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") + +_MINICPMO_SERVER = [ + pytest.param( + OmniServerParams( + model="openbmb/MiniCPM-o-4_5", + stage_config_path=_MINICPMO_DEPLOY, + use_stage_cli=True, + server_args=[ + "--trust-remote-code", + "--no-async-chunk", + ], + ), + id="minicpmo_4_5", + ) +] + + +def _system_prompt() -> dict[str, object]: + return { + "role": "system", + "content": [ + { + "type": "text", + "text": ( + "You are MiniCPM-o 4.5, a virtual human capable of perceiving " + "auditory and visual inputs, as well as generating text and speech." + ), + } + ], + } + + +def _prompt() -> str: + return "What is the capital of China? Answer in 20 words." + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True) +def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None: + """ + Test that invalid audio format is properly rejected. + Deploy Setting: default 2GPU + Input Modal: text + invalid audio format + Output Modal: error + """ + fake_base64 = "AAAA" # not a real audio file + + responses = openai_client.send_chat_completions_http_request( + { + "json": { + "model": omni_server.model, + "messages": dummy_messages_from_mix_data( + system_prompt=_system_prompt(), + audio_data_url=f"data:audio/invalid;base64,{fake_base64}", + content_text=_prompt(), + ), + "stream": True, + }, + }, + err_code=400, + err_message="Invalid or unsupported audio file", + ) + assert not responses[0].success diff --git a/tests/e2e/offline_inference/test_minicpmo_4_5.py b/tests/e2e/offline_inference/test_minicpmo_4_5.py new file mode 100644 index 000000000..ee1dc4d01 --- /dev/null +++ b/tests/e2e/offline_inference/test_minicpmo_4_5.py @@ -0,0 +1,118 @@ +""" +E2E offline tests for MiniCPM-o 4.5 model with multimodal input and audio / text output. +""" + +import os + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video +from tests.helpers.stage_config import get_deploy_config_path + +models = ["openbmb/MiniCPM-o-4_5"] + +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") + + +test_params = [(model, None, {"deploy_config": _CI_DEPLOY, "trust_remote_code": True}) for model in models] + + +def get_question(prompt_type: str = "text") -> str: + prompts = { + "text": "What is the capital of China? Answer in 20 words.", + "audio": "Describe the audio briefly.", + "image": "What color are the squares in this image?", + "video": "Describe the video briefly.", + "mix": "Describe what is in the image and audio.", + } + return prompts.get(prompt_type, prompts["text"]) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_text_to_text(omni_runner, omni_runner_handler) -> None: + """Test processing text, generating text output.""" + request_config = {"prompts": get_question("text"), "modalities": ["text"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_audio_to_text(omni_runner, omni_runner_handler) -> None: + """Test processing audio, generating text output.""" + audio = generate_synthetic_audio(1, 1, 16000)["np_array"] + if len(audio.shape) == 2: + audio = audio.squeeze() + request_config = {"prompts": get_question("audio"), "audios": (audio, 16000), "modalities": ["text"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_image_to_text(omni_runner, omni_runner_handler) -> None: + """Test processing image, generating text output.""" + image = generate_synthetic_image(16, 16)["np_array"] + request_config = {"prompts": get_question("image"), "images": image, "modalities": ["text"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_video_to_text(omni_runner, omni_runner_handler) -> None: + """Test processing video, generating text output.""" + video = generate_synthetic_video(24, 24, 20)["np_array"] + request_config = {"prompts": get_question("video"), "videos": video, "modalities": ["text"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_text_to_audio(omni_runner, omni_runner_handler) -> None: + """Test processing text, generating audio output through the talker token2wav path.""" + request_config = {"prompts": get_question("text"), "modalities": ["audio"]} + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: + """Test processing mixed modalities (image + audio), generating audio output.""" + audio = generate_synthetic_audio(1, 1, 16000)["np_array"] + if len(audio.shape) == 2: + audio = audio.squeeze() + image = generate_synthetic_image(16, 16)["np_array"] + request_config = { + "prompts": get_question("mix"), + "audios": (audio, 16000), + "images": image, + "modalities": ["audio"], + } + omni_runner_handler.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_runner", test_params, indirect=True) +def test_video_to_audio(omni_runner, omni_runner_handler) -> None: + """Test processing video, generating audio output.""" + video = generate_synthetic_video(24, 24, 20)["np_array"] + request_config = {"prompts": get_question("video"), "videos": video, "modalities": ["audio"]} + omni_runner_handler.send_omni_request(request_config) diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py new file mode 100644 index 000000000..7d2c7948e --- /dev/null +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -0,0 +1,233 @@ +""" +E2E Online tests for MiniCPM-o 4.5 model with multimodal input and audio / text output. + +MiniCPM-o 4.5 has ``async_chunk: false``, ``max_num_seqs: 1`` on both stages, +and the vocoder runs in-process inside the talker stage rather than as a separate +Code2Wav stage. +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video +from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data +from tests.helpers.stage_config import get_deploy_config_path + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +_MODEL = "openbmb/MiniCPM-o-4_5" +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") + +test_params = [ + pytest.param( + OmniServerParams( + model=_MODEL, + stage_config_path=_CI_DEPLOY, + use_stage_cli=True, + server_args=[ + "--trust-remote-code", + "--no-async-chunk", + ], + ), + id="default", + ) +] + + +def get_system_prompt(): + return { + "role": "system", + "content": [ + { + "type": "text", + "text": ( + "You are MiniCPM-o 4.5, a virtual human capable of perceiving " + "auditory and visual inputs, as well as generating text and speech." + ), + } + ], + } + + +def get_prompt(prompt_type: str = "text_only") -> str: + prompts = { + "text_only": "What is the capital of China? Answer in 20 words.", + "mix": "What is recited in the audio? What is in this image? Describe the video briefly.", + "text_image": "What color are the squares in this image?", + } + return prompts.get(prompt_type, prompts["text_only"]) + + +def get_max_batch_size(size_type="few"): + batch_sizes = {"few": 5, "medium": 100, "large": 256} + return batch_sizes.get(size_type, 5) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_text_001(omni_server, openai_client) -> None: + """ + Test text-only input generating text output via OpenAI API. + Deploy Setting: default 2GPU + Input Modal: text + Output Modal: text + Input Setting: stream=False + """ + messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": False, + "modalities": ["text"], + "key_words": {"text": ["Beijing"]}, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_audio_001(omni_server, openai_client) -> None: + """ + Test text-only input generating text + audio output via OpenAI API. + This exercises the talker TTS region detection and in-process token2wav vocoder. + Deploy Setting: default 2GPU + Input Modal: text + Output Modal: text + audio + Input Setting: stream=True + """ + messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + "key_words": {"audio": ["Beijing"]}, + } + + openai_client.send_omni_request(request_config) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_audio_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test audio input generating text + audio output via OpenAI API. + Deploy Setting: default 2GPU + Input Modal: text + audio + Output Modal: text + audio + Input Setting: stream=True + """ + audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + audio_data_url=audio_data_url, + content_text=get_prompt("mix"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_image_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test image input generating text + audio output via OpenAI API. + Deploy Setting: default 2GPU + Input Modal: text + image + Output Modal: text + audio + Input Setting: stream=True + """ + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(24, 24)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + image_data_url=image_data_url, + content_text=get_prompt("text_image"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@pytest.mark.full_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_video_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test video input generating text + audio output via OpenAI API. + Deploy Setting: default 2GPU + Input Modal: text + video + Output Modal: text + audio + Input Setting: stream=True + """ + video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(24, 24, 20)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + video_data_url=video_data_url, + content_text=get_prompt("mix"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_mix_to_text_audio_001(omni_server, openai_client) -> None: + """ + Test multi-modal input (text + audio + video + image) generating text + audio output. + Deploy Setting: default 2GPU + Input Modal: text + audio + video + image + Output Modal: text + audio + Input Setting: stream=True + """ + video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(24, 24, 20)['base64']}" + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(24, 24)['base64']}" + audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + video_data_url=video_data_url, + image_data_url=image_data_url, + audio_data_url=audio_data_url, + content_text=get_prompt("mix"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py new file mode 100644 index 000000000..8db6cd92b --- /dev/null +++ b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py @@ -0,0 +1,178 @@ +""" +E2E Online expansion tests for MiniCPM-o 4.5 covering modality combinations, +in-process token2wav vocoder behavior, max_num_seqs=1 serial execution, +and edge-case validation. +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import generate_synthetic_video +from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data +from tests.helpers.stage_config import get_deploy_config_path + +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +_MODEL = "openbmb/MiniCPM-o-4_5" +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") + +test_params = [ + pytest.param( + OmniServerParams( + model=_MODEL, + stage_config_path=_CI_DEPLOY, + use_stage_cli=True, + server_args=[ + "--trust-remote-code", + ], + ), + id="default", + ) +] + + +def get_system_prompt(): + return { + "role": "system", + "content": [ + { + "type": "text", + "text": ( + "You are MiniCPM-o 4.5, a virtual human capable of perceiving " + "auditory and visual inputs, as well as generating text and speech." + ), + } + ], + } + + +def get_prompt(prompt_type: str = "text_only") -> str: + prompts = { + "text_only": "What is the capital of China? Answer in 20 words.", + "mix": "What is recited in the audio? What is in this image? Describe the video briefly.", + } + return prompts.get(prompt_type, prompts["text_only"]) + + +def get_max_batch_size(size_type="few"): + batch_sizes = {"few": 5, "medium": 100, "large": 256} + return batch_sizes.get(size_type, 5) + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_video_to_text_001(omni_server, openai_client) -> None: + """ + Test text + video input generating text output. + Deploy Setting: default 2GPU + Input Modal: text + video + Output Modal: text + Input Setting: stream=False + """ + video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(24, 24, 20)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + video_data_url=video_data_url, + content_text=get_prompt("mix"), + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": False, + "modalities": ["text"], + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_sequential_requests_independent(omni_server, openai_client) -> None: + """ + Verify that sequential requests produce independent results and state + does not leak between requests. MiniCPM-o 4.5 has max_num_seqs=1 so + requests are processed one-at-a-time; the second request must not + receive the first request's audio or token state. + Deploy Setting: default 2GPU + Input Modal: text (two different prompts) + Output Modal: text + audio (both) + """ + messages_1 = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="What is the capital of France?", + ) + messages_2 = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="What is the capital of China?", + ) + + # Send first request and ensure it completes + openai_client.send_omni_request( + { + "model": omni_server.model, + "messages": messages_1, + "stream": True, + }, + request_num=1, + ) + + # Send second request with different prompt; must produce its own output + openai_client.send_omni_request( + { + "model": omni_server.model, + "messages": messages_2, + "stream": True, + "key_words": {"text": ["Beijing"]}, + }, + request_num=1, + ) + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: + """ + Test text input generating a longer audio output to exercise the + token2wav decoder across multiple frames. + Deploy Setting: default 2GPU + Input Modal: text (longer prompt) + Output Modal: text + audio + Input Setting: stream=True + """ + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="Tell me a short story about a cat in about 50 words.", + ) + + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + "key_words": {"audio": ["cat"]}, + } + + openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) + + +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_chinese_text_to_audio(omni_server, openai_client) -> None: + """ + Test Chinese text input generating audio output. + """ + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + content_text="北京，中国的首都，是一座融合了长城等历史地点与现代建筑的国际化大都市，充满了独特的文化与活力。请重复这句话。", + ) + request_config = { + "model": omni_server.model, + "messages": messages, + "stream": True, + "key_words": {"text": ["北京"]}, + } + openai_client.send_omni_request(request_config) diff --git a/tests/entrypoints/test_stream_finish_reason.py b/tests/entrypoints/test_stream_finish_reason.py index ebbca1917..63ec9f858 100644 --- a/tests/entrypoints/test_stream_finish_reason.py +++ b/tests/entrypoints/test_stream_finish_reason.py @@ -403,6 +403,90 @@ async def test_declared_modality_not_produced_text_finish_suppressed(): assert ch["finish_reason"] is None +# --------------------------------------------------------------------------- +# Tests: audio chunks that carry no waveform +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_audio_chunk_without_waveform_keeps_stream_alive(): + """An audio message with no PCM must not break the stream. + + ``_create_audio_choice`` used to return an ErrorResponse here, which the + stream generator then iterated as a list of choices; the pydantic model + yields (field, value) tuples, so the loop raised + ``AttributeError: 'tuple' object has no attribute 'finish_reason'``. + """ + serving_chat = _build_serving_chat() + request = _make_request(modalities=["text", "audio"]) + + empty_audio = _make_audio_omni_output() + empty_audio.request_output.outputs[0].multimodal_output = {"audio": []} + + def create_audio_choice(omni_res, role, request, stream=False): + return OmniOpenAIServingChat._create_audio_choice(serving_chat, omni_res, role, request, stream=stream) + + serving_chat._create_audio_choice = create_audio_choice + + async def result_generator(): + yield _make_text_omni_output(text="hi", token_ids=[10], finish_reason=None) + yield _make_text_omni_output(text="!", token_ids=[11], finish_reason="stop") + yield empty_audio + + raw_lines = await _collect_stream( + serving_chat.chat_completion_stream_generator( + request=request, + result_generator=result_generator(), + request_id="test-req", + model_name="test-model", + conversation=[], + tokenizer=MagicMock(), + request_metadata=MagicMock(), + ) + ) + + assert not any("Error in chat completion stream generator" in line for line in raw_lines) + chunks = _parse_sse_chunks(raw_lines) + finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] + assert finish_reasons.count("stop") == 1, f"Expected 1 stop, got {finish_reasons}" + assert finish_reasons[-1] == "stop" + + +@pytest.mark.asyncio +async def test_audio_choice_error_response_is_not_iterated_as_choices(): + """Any ErrorResponse from the audio path is skipped, not iterated.""" + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + serving_chat = _build_serving_chat() + request = _make_request(modalities=["text", "audio"]) + + serving_chat._create_audio_choice = MagicMock( + side_effect=lambda omni_res, role, request, stream=False: serving_chat._create_error_response("boom") + ) + + async def result_generator(): + yield _make_text_omni_output(text="hi", token_ids=[10], finish_reason="stop") + yield _make_audio_omni_output() + + raw_lines = await _collect_stream( + serving_chat.chat_completion_stream_generator( + request=request, + result_generator=result_generator(), + request_id="test-req", + model_name="test-model", + conversation=[], + tokenizer=MagicMock(), + request_metadata=MagicMock(), + ) + ) + + assert isinstance(serving_chat._create_error_response("boom"), ErrorResponse) + assert not any("AttributeError" in line for line in raw_lines) + chunks = _parse_sse_chunks(raw_lines) + finish_reasons = [ch["finish_reason"] for c in chunks for ch in c.get("choices", [])] + assert finish_reasons.count("stop") == 1, f"Expected 1 stop, got {finish_reasons}" + + # --------------------------------------------------------------------------- # Tests: voice/speaker parameter compatibility # --------------------------------------------------------------------------- diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 5d1d9a3d1..39baf1ce0 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -2424,6 +2424,10 @@ class OmniRunner: video_padding_token = "<|VIDEO|>" image_padding_token = "<|IMAGE|>" audio_padding_token = "<|AUDIO|>" + # Default wrapping for Qwen-style models (bos/eos around the placeholder). + audio_fmt = "<|audio_bos|>{p}<|audio_eos|>" + image_fmt = "<|vision_bos|>{p}<|vision_eos|>" + video_fmt = "<|vision_bos|>{p}<|vision_eos|>" if "Qwen3-Omni-30B-A3B-Instruct" in self.model_name: video_padding_token = "<|video_pad|>" image_padding_token = "<|image_pad|>" @@ -2432,6 +2436,15 @@ class OmniRunner: video_padding_token = "