diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 86eec1163..a2f78f05c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -9,6 +9,14 @@ steps: agents: queue: "cpu_queue_premerge_us_east_1" + - label: "Test on NPU" + depends_on: ~ + key: npu-test + commands: + - ".buildkite/scripts/hardware_ci/run_npu_test.sh" + agents: + queue: "ascend" + - label: "Simple Unit Test" commands: - ".buildkite/scripts/simple_test.sh" diff --git a/.buildkite/scripts/hardware_ci/run_npu_test.sh b/.buildkite/scripts/hardware_ci/run_npu_test.sh new file mode 100644 index 000000000..fbe24badb --- /dev/null +++ b/.buildkite/scripts/hardware_ci/run_npu_test.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# This script build the Ascend NPU docker image and run the offline inference inside the container. +# It serves a sanity check for compilation and basic model usage. +set -ex + +# Base ubuntu image with basic ascend development libraries and python installed +VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git" +BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2" +image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}" +# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}" +container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" + +# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards +agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}') +echo "agent_idx: ${agent_idx}" +builder_name="cachebuilder${agent_idx}" +builder_cache_dir="/mnt/docker-cache${agent_idx}" +mkdir -p ${builder_cache_dir} + +# Try building the docker image +cat <=6.0 pytest-cov modelscope + +COPY . . + +# Install vllm-omni +WORKDIR /workspace +ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git +ARG VLLM_OMNI_TAG=main +ARG BUILDKITE_PULL_REQUEST +ARG BUILDKITE_PULL_REQUEST_REPO +RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \ + if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \ + echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \ + git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \ + cd /workspace/vllm-omni && \ + git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \ + git checkout pr-\$BUILDKITE_PULL_REQUEST; \ + else \ + echo "Not a PR build, using main branch" && \ + git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \ + fi + +RUN --mount=type=cache,target=/root/.cache/pip \ + export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ + python3 -m pip install -v -e /workspace/vllm-omni/ + +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn + +WORKDIR /workspace/vllm-omni +CMD ["/bin/bash"] + +EOF + +# Setup cleanup +remove_docker_container() { + docker rm -f "${container_name}" || true; + docker image rm -f "${image_name}" || true; + docker system prune -f || true; +} +trap remove_docker_container EXIT + +# Generate corresponding --device args based on BUILDKITE_AGENT_NAME +# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1. +# e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards. +# returns --device /dev/davinci0 --device /dev/davinci1 +parse_and_gen_devices() { + local input="$1" + local index cards_num + if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then + index="${BASH_REMATCH[1]}" + cards_num="${BASH_REMATCH[2]}" + else + echo "parse error" >&2 + return 1 + fi + + local devices="" + local i=0 + while (( i < cards_num )); do + local dev_idx=$(((index - 1)*cards_num + i )) + devices="$devices --device /dev/davinci${dev_idx}" + ((i++)) + done + + # trim leading space + devices="${devices#"${devices%%[![:space:]]*}"}" + # Output devices: assigned to the caller variable + printf '%s' "$devices" +} + +devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1 + +# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware. +# This test checks whether the OOT platform interface is functioning properly in conjunction with +# the hardware plugin vllm-ascend. +hf_model_cache_dir=/mnt/hf_cache${agent_idx} +ms_model_cache_dir=/mnt/modelscope${agent_idx} +mkdir -p ${hf_model_cache_dir} +mkdir -p ${ms_model_cache_dir} +docker run \ + --init \ + ${devices} \ + --device /dev/davinci_manager \ + --device /dev/devmm_svm \ + --device /dev/hisi_hdc \ + -v /usr/local/dcmi:/usr/local/dcmi \ + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ + -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \ + -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \ + -v /etc/ascend_install.info:/etc/ascend_install.info \ + -v ${hf_model_cache_dir}:/root/.cache/huggingface \ + -v ${ms_model_cache_dir}:/root/.cache/modelscope \ + --network host \ + --entrypoint="" \ + --name "${container_name}" \ + "${image_name}" \ + bash -c ' + set -e + VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py +' diff --git a/docker/Dockerfile.ci.npu b/docker/Dockerfile.ci.npu new file mode 100644 index 000000000..cdf7a70f3 --- /dev/null +++ b/docker/Dockerfile.ci.npu @@ -0,0 +1,15 @@ +ARG VLLM_ASCEND_IMAGE=quay.nju.edu.cn/ascend/vllm-ascend +ARG VLLM_ASCEND_TAG=v0.11.0rc2 +FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} + +ARG APP_DIR=/vllm-workspace/vllm-omni +WORKDIR ${APP_DIR} + +COPY . . + +# Install vllm-omni with dev dependencies +RUN pip install --no-cache-dir -e ".[dev]" + +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn + +ENTRYPOINT [] diff --git a/tests/e2e/offline_inference/stage_configs/npu/qwen2_5_omni_ci.yaml b/tests/e2e/offline_inference/stage_configs/npu/qwen2_5_omni_ci.yaml new file mode 100644 index 000000000..d2aae1c0d --- /dev/null +++ b/tests/e2e/offline_inference/stage_configs/npu/qwen2_5_omni_ci.yaml @@ -0,0 +1,104 @@ +# stage config for running qwen2.5-omni with architecture of OmniLLM. + +# This config is optimized for CI e2e tests. +stage_args: + - stage_id: 0 + runtime: + process: true # Run this stage in a separate process + devices: "0" + max_batch_size: 1 + engine_args: + model_stage: thinker + model_arch: Qwen2_5OmniForConditionalGeneration + worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + max_model_len: 896 + max_num_batched_tokens: 896 + max_num_seqs: 1 + gpu_memory_utilization: 0.8 + skip_mm_profiling: true + enforce_eager: true # Now we only support eager mode + trust_remote_code: true + engine_output_type: latent + enable_prefix_caching: false + is_comprehension: true + final_output: true + final_output_type: text + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 128 + seed: 42 + detokenize: True + repetition_penalty: 1.1 + - stage_id: 1 + runtime: + process: true + devices: "1" + max_batch_size: 1 + engine_args: + model_stage: talker + model_arch: Qwen2_5OmniForConditionalGeneration + worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + max_model_len: 896 + max_num_batched_tokens: 896 + max_num_seqs: 1 + gpu_memory_utilization: 0.8 + skip_mm_profiling: true + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + engine_output_type: latent + engine_input_source: [0] + custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker + default_sampling_params: + temperature: 0.9 + top_p: 0.8 + top_k: 40 + max_tokens: 128 + seed: 42 + detokenize: True + repetition_penalty: 1.05 + stop_token_ids: [8294] + - stage_id: 2 + runtime: + process: true + devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU + max_batch_size: 1 + engine_args: + model_stage: code2wav + model_arch: Qwen2_5OmniForConditionalGeneration + worker_cls: vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker + scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler + gpu_memory_utilization: 0.15 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + engine_output_type: audio + engine_input_source: [1] + final_output: true + final_output_type: audio + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 128 + seed: 42 + detokenize: True + repetition_penalty: 1.1 + +# Top-level runtime config (concise): default windows and stage edges +runtime: + enabled: true + defaults: + window_size: -1 # Simplified: trigger downstream only after full upstream completion + max_inflight: 1 # Simplified: process serially within each stage + edges: + - from: 0 # thinker → talker: trigger only after receiving full input (-1) + to: 1 + window_size: -1 + - from: 1 # talker → code2wav: trigger only after receiving full input (-1) + to: 2 + window_size: -1 diff --git a/tests/e2e/offline_inference/test_qwen2_5_omni.py b/tests/e2e/offline_inference/test_qwen2_5_omni.py index 72478cfd4..c47f79a64 100644 --- a/tests/e2e/offline_inference/test_qwen2_5_omni.py +++ b/tests/e2e/offline_inference/test_qwen2_5_omni.py @@ -10,18 +10,24 @@ import pytest from vllm.assets.audio import AudioAsset from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset +from vllm.envs import VLLM_USE_MODELSCOPE from vllm.multimodal.image import convert_image_mode +from vllm_omni.utils import is_npu + from .conftest import OmniRunner from .utils import create_new_process_for_each_test models = ["Qwen/Qwen2.5-Omni-3B"] -# CI stage config optimized for 24GB GPU (L4/RTX3090) -stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen2_5_omni_ci.yaml")] +# CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU +if is_npu(): + stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml") +else: + stage_config = str(Path(__file__).parent / "stage_configs" / "qwen2_5_omni_ci.yaml") # Create parameter combinations for model and stage config -test_params = [(model, stage_config) for model in models for stage_config in stage_configs] +test_params = [(model, stage_config) for model in models] @pytest.mark.core_model @@ -36,7 +42,11 @@ def test_mixed_modalities_to_audio(omni_runner: type[OmniRunner], test_config: t audio = AudioAsset("mary_had_lamb").audio_and_sample_rate audio = (audio[0][: 16000 * 5], audio[1]) # Trim to first 5 seconds image = convert_image_mode(ImageAsset("cherry_blossom").pil_image.resize((128, 128)), "RGB") - video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays + if not VLLM_USE_MODELSCOPE: + video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays + else: + # modelscope can't access raushan-testing-hf/videos-test, skip video input temporarily + video = None outputs = runner.generate_multimodal( prompts=question, diff --git a/tests/e2e/offline_inference/test_t2i_model.py b/tests/e2e/offline_inference/test_t2i_model.py index 770978366..fafd98e55 100644 --- a/tests/e2e/offline_inference/test_t2i_model.py +++ b/tests/e2e/offline_inference/test_t2i_model.py @@ -5,6 +5,8 @@ from pathlib import Path import pytest import torch +from vllm_omni.utils.platform_utils import is_npu + # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] if str(REPO_ROOT) not in sys.path: @@ -14,8 +16,15 @@ from vllm_omni import Omni os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" + models = ["Tongyi-MAI/Z-Image-Turbo", "riverclouds/qwen_image_random"] +# NPU still can't run Tongyi-MAI/Z-Image-Turbo properly +# Modelscope can't find riverclouds/qwen_image_random +# TODO: When NPU support is ready, remove this branch. +if is_npu(): + models = ["Qwen/Qwen-Image"] + @pytest.mark.parametrize("model_name", models) def test_diffusion_model(model_name: str):