diff --git a/docker/Dockerfile.npu b/docker/Dockerfile.npu index 95774dd25..29ea95cc3 100644 --- a/docker/Dockerfile.npu +++ b/docker/Dockerfile.npu @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e +ARG VLLM_ASCEND_TAG=v0.24.0 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} # WORKDIR /vllm-workspace/vllm diff --git a/docker/Dockerfile.npu.a3 b/docker/Dockerfile.npu.a3 index ea8882a38..90036d345 100644 --- a/docker/Dockerfile.npu.a3 +++ b/docker/Dockerfile.npu.a3 @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e-a3 +ARG VLLM_ASCEND_TAG=v0.24.0-a3 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} # WORKDIR /vllm-workspace/vllm diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index 2c027992e..f8746d2c8 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e +ARG VLLM_ASCEND_TAG=v0.24.0 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} ARG APP_DIR=/vllm-workspace/vllm-omni diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index 4e0de0a1d..5e5848d39 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e-a3 +ARG VLLM_ASCEND_TAG=v0.24.0-a3 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} ARG APP_DIR=/vllm-workspace/vllm-omni diff --git a/docs/getting_started/installation/npu/npu.inc.md b/docs/getting_started/installation/npu/npu.inc.md index c5b13dd73..87d479b49 100644 --- a/docs/getting_started/installation/npu/npu.inc.md +++ b/docs/getting_started/installation/npu/npu.inc.md @@ -1,19 +1,16 @@ # --8<-- [start:requirements] -For detailed hardware and software requirements, please refer to the [vllm-ascend installation documentation](https://docs.vllm.ai/projects/ascend/en/latest/installation.html). +For detailed hardware and software requirements, please refer to the [vLLM-Ascend installation documentation](https://docs.vllm.ai/projects/ascend/en/latest/installation.html). # --8<-- [end:requirements] # --8<-- [start:installation-release] -The recommended way to use vLLM-Omni on NPU is through the vllm-ascend pre-built Docker images: +The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built Docker images: ```bash -# Update the vllm-ascend image -# Atlas A2: -# export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1 -# Atlas A3: -# export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1-a3 -export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1 +# vLLM-Ascend image +# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.24.0 +# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.24.0-a3 docker run --rm \ --name vllm-omni-npu \ --shm-size=1g \ @@ -31,11 +28,11 @@ docker run --rm \ -v /etc/ascend_install.info:/etc/ascend_install.info \ -v /root/.cache:/root/.cache \ -p 8000:8000 \ - -it $IMAGE bash + -it quay.io/atlas-ci/vllm-ascend:0.24.0-a3 bash # Inside the container, install vLLM-Omni from source cd /vllm-workspace -git clone -b v0.18.0 https://github.com/vllm-project/vllm-omni.git +git clone -b v0.24.0 https://github.com/vllm-project/vllm-omni.git cd vllm-omni pip install -v -e . --no-build-isolation # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e . @@ -87,14 +84,11 @@ Supported images as following. Here's an example deployment command that has been verified on 4 x NPUs: ```bash -# Atlas A2: -# export IMAGE=quay.io/ascend/vllm-omni:v0.18.0 -# Atlas A3: -# export IMAGE=quay.io/ascend/vllm-omni:v0.18.0-a3 -export IMAGE=quay.io/ascend/vllm-omni:v0.18.0 +# Atlas A2: quay.io/ascend/vllm-omni:v0.24.0 +# Atlas A3: quay.io/ascend/vllm-omni:v0.24.0-a3 docker run --rm \ - --name vllm-omni-npu \ - --shm-size=1g \ + --name vllm-omni-a3 \ + --shm-size=4g \ --device /dev/davinci0 \ --device /dev/davinci1 \ --device /dev/davinci2 \ @@ -106,10 +100,17 @@ docker run --rm \ -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \ -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \ + --device /dev/davinci_manager \ + --device /dev/devmm_svm \ + --device /dev/hisi_hdc \ + -v /usr/local/dcmi:/usr/local/dcmi \ + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ + -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \ + -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \ -v /etc/ascend_install.info:/etc/ascend_install.info \ - -v /root/.cache:/root/.cache \ - -p 8000:8000 \ - -it $IMAGE bash + -v ~/.cache:/root/.cache \ + -p 8091:8091 \ + -it quay.io/ascend/vllm-omni:v0.24.0-a3 bash ``` !!! tip diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index c66d493c2..b469234c1 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -479,7 +479,6 @@ class NPUARModelRunner(OmniNPUModelRunner): logits_indices, spec_decode_metadata, total_num_scheduled_tokens, - num_scheduled_tokens_compressed_list, ) = self._prepare_inputs( scheduler_output, num_scheduled_tokens_np, @@ -576,7 +575,12 @@ class NPUARModelRunner(OmniNPUModelRunner): # Another possible condition is num_tokens_padded != num_tokens_unpadded # but this scope is way too big and the consequences are unpredictable num_reqs_padded = self._pad_query_start_loc_for_fia( - num_tokens_padded, num_reqs_padded, num_reqs, cudagraph_mode, batch_desc.num_reqs + self.query_start_loc, + num_tokens_padded, + num_reqs_padded, + num_reqs, + cudagraph_mode, + batch_desc.num_reqs, ) (attn_metadata, spec_decode_common_attn_metadata) = self._build_attention_metadata( diff --git a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py index df8b92bf5..ea03842b4 100644 --- a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py @@ -164,7 +164,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): logits_indices, spec_decode_metadata, total_num_scheduled_tokens, - num_scheduled_tokens_compressed_list, ) = self._prepare_inputs( scheduler_output, num_scheduled_tokens_np, @@ -261,7 +260,12 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): # Another possible condition is num_tokens_padded != num_tokens_unpadded # but this scope is way too big and the consequences are unpredictable num_reqs_padded = self._pad_query_start_loc_for_fia( - num_tokens_padded, num_reqs_padded, num_reqs, cudagraph_mode, batch_desc.num_reqs + self.query_start_loc, + num_tokens_padded, + num_reqs_padded, + num_reqs, + cudagraph_mode, + batch_desc.num_reqs, ) (attn_metadata, spec_decode_common_attn_metadata) = self._build_attention_metadata( @@ -729,7 +733,12 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): if not profile_cpp: num_reqs_padded = self._pad_query_start_loc_for_fia( - num_tokens_padded, num_reqs_padded, num_reqs, cudagraph_runtime_mode, batch_desc.num_reqs + self.query_start_loc, + num_tokens_padded, + num_reqs_padded, + num_reqs, + cudagraph_runtime_mode, + batch_desc.num_reqs, ) pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL diff --git a/vllm_omni/platforms/npu/worker/npu_model_runner.py b/vllm_omni/platforms/npu/worker/npu_model_runner.py index 3b9b4937a..671c7b7e5 100644 --- a/vllm_omni/platforms/npu/worker/npu_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_model_runner.py @@ -193,7 +193,12 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): if not profile_cpp: num_reqs_padded = self._pad_query_start_loc_for_fia( - num_tokens_padded, num_reqs_padded, num_reqs, cudagraph_runtime_mode, batch_desc.num_reqs + self.query_start_loc, + num_tokens_padded, + num_reqs_padded, + num_reqs, + cudagraph_runtime_mode, + batch_desc.num_reqs, ) pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL