diff --git a/.buildkite/pipeline-npu.yaml b/.buildkite/pipeline-npu.yaml index 161b0e944..d035ffc94 100644 --- a/.buildkite/pipeline-npu.yaml +++ b/.buildkite/pipeline-npu.yaml @@ -78,8 +78,89 @@ steps: echo "--- Image pushed successfully" echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + - label: ":buildkit: Build and Push NPU Test Image - A3" + key: image-build-a3 + agents: + queue: "ascend-a2b3" + resource_class: "npu-1" + plugins: + - kubernetes: + metadata: + annotations: + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" + vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" + vault.hashicorp.com/agent-inject-perms-config.json: "0400" + vault.hashicorp.com/agent-inject-perms-key.pem: "0400" + vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-group: "1000" + vault.hashicorp.com/agent-run-as-user: "1000" + vault.hashicorp.com/agent-service-account-token-volume-name: token-vol + vault.hashicorp.com/role: ascend-gha-runners + vault.hashicorp.com/secret-volume-path: /home/user/.docker/ + vault.hashicorp.com/tls-skip-verify: "true" + podSpecPatch: + volumes: + - name: token-vol + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api + expirationSeconds: 600 + path: token + env: + VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3" + IMAGE_NAME: "vllm-omni-ci-npu" + IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" + BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" + command: | + set -ex + + echo "--- Building and pushing NPU Test Image" + echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + echo "buildkitd address: $${BUILDKITD_ADDR}" + + if ! command -v buildctl &> /dev/null; then + echo "Installing buildctl..." + mkdir -p /tmp/buildkit + BUILDKIT_VERSION="v0.29.0" + wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz + tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit + cp /tmp/buildkit/bin/buildctl /usr/local/bin/ + fi + + export DOCKER_CONFIG=/home/user/.docker + buildctl \ + --addr="$${BUILDKITD_ADDR}" \ + --tlscacert=/home/user/.docker/ca.pem \ + --tlscert=/home/user/.docker/cert.pem \ + --tlskey=/home/user/.docker/key.pem \ + build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=./docker \ + --opt filename=Dockerfile.npu.ci.a3 \ + --secret id=dockerconfig,src=/home/user/.docker/config.json \ + --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \ + --progress=plain + + echo "--- Image pushed successfully" + echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}" + - label: "Upload Ready Pipeline" - depends_on: image-build + depends_on: + - image-build + - image-build-a3 key: upload-ready-pipeline if: >- build.branch != "main" && ( @@ -94,7 +175,9 @@ steps: # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) - label: "Upload Nightly Pipeline" - depends_on: image-build + depends_on: + - image-build + - image-build-a3 key: upload-nightly-pipeline if: >- (build.branch == "main" && build.env("NIGHTLY") == "1") || diff --git a/.buildkite/test-npu-nightly.yaml b/.buildkite/test-npu-nightly.yaml index 2c2372f4c..4dbda17f9 100644 --- a/.buildkite/test-npu-nightly.yaml +++ b/.buildkite/test-npu-nightly.yaml @@ -22,3 +22,31 @@ steps: HF_TOKEN: "${HF_TOKEN}" commands: - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model" + - group: ":card_index_dividers: TTS Model Test" + key: nightly-tts-test-group + depends_on: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" + steps: + - label: ":full_moon: TTS · Perf Test" + agents: + queue: "ascend-a3" + resource_class: "npu-2" + image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" + plugins: + - kubernetes: + podSpecPatch: + imagePullSecrets: + - name: swr-secret + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index f5c3cdfaf..a3cede3a7 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -6,6 +6,8 @@ ARG APP_DIR=/vllm-workspace/vllm-omni WORKDIR ${APP_DIR} COPY . . +RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list + RUN apt-get update && \ apt-get install -y --no-install-recommends \ ffmpeg \ diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index 6a8ac4678..1369d8c85 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -6,6 +6,8 @@ ARG APP_DIR=/vllm-workspace/vllm-omni WORKDIR ${APP_DIR} COPY . . +RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list + RUN apt-get update && \ apt-get install -y --no-install-recommends \ ffmpeg \ diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 3bbf60935..85236eab7 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 }