[CI][NPU]: Add nightly performance test for NPU - Qwen3-TTS on A3 (#5158)
Signed-off-by: Weiming Liao <liaowm5@gmail.com> Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com>
This commit is contained in:
Weiming Liaoandwangyu authored and GitHub committed
1 parent
be517db18a
commit
7aa5c9a090
5 files changed
+119
-3
No files matched your search
@@ -78,8 +78,89 @@ steps:
|
||||
echo "--- Image pushed successfully"
|
||||
echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
|
||||
- label: ":buildkit: Build and Push NPU Test Image - A3"
|
||||
key: image-build-a3
|
||||
agents:
|
||||
queue: "ascend-a2b3"
|
||||
resource_class: "npu-1"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
metadata:
|
||||
annotations:
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-config.json: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-run-as-group: "1000"
|
||||
vault.hashicorp.com/agent-run-as-user: "1000"
|
||||
vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
|
||||
vault.hashicorp.com/role: ascend-gha-runners
|
||||
vault.hashicorp.com/secret-volume-path: /home/user/.docker/
|
||||
vault.hashicorp.com/tls-skip-verify: "true"
|
||||
podSpecPatch:
|
||||
volumes:
|
||||
- name: token-vol
|
||||
projected:
|
||||
defaultMode: 420
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
audience: api
|
||||
expirationSeconds: 600
|
||||
path: token
|
||||
env:
|
||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
|
||||
IMAGE_NAME: "vllm-omni-ci-npu"
|
||||
IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
|
||||
BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
|
||||
command: |
|
||||
set -ex
|
||||
|
||||
echo "--- Building and pushing NPU Test Image"
|
||||
echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
echo "buildkitd address: $${BUILDKITD_ADDR}"
|
||||
|
||||
if ! command -v buildctl &> /dev/null; then
|
||||
echo "Installing buildctl..."
|
||||
mkdir -p /tmp/buildkit
|
||||
BUILDKIT_VERSION="v0.29.0"
|
||||
wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
|
||||
tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
|
||||
cp /tmp/buildkit/bin/buildctl /usr/local/bin/
|
||||
fi
|
||||
|
||||
export DOCKER_CONFIG=/home/user/.docker
|
||||
buildctl \
|
||||
--addr="$${BUILDKITD_ADDR}" \
|
||||
--tlscacert=/home/user/.docker/ca.pem \
|
||||
--tlscert=/home/user/.docker/cert.pem \
|
||||
--tlskey=/home/user/.docker/key.pem \
|
||||
build \
|
||||
--frontend dockerfile.v0 \
|
||||
--local context=. \
|
||||
--local dockerfile=./docker \
|
||||
--opt filename=Dockerfile.npu.ci.a3 \
|
||||
--secret id=dockerconfig,src=/home/user/.docker/config.json \
|
||||
--output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
|
||||
--progress=plain
|
||||
|
||||
echo "--- Image pushed successfully"
|
||||
echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
|
||||
- label: "Upload Ready Pipeline"
|
||||
depends_on: image-build
|
||||
depends_on:
|
||||
- image-build
|
||||
- image-build-a3
|
||||
key: upload-ready-pipeline
|
||||
if: >-
|
||||
build.branch != "main" && (
|
||||
@@ -94,7 +175,9 @@ steps:
|
||||
|
||||
# L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
- label: "Upload Nightly Pipeline"
|
||||
depends_on: image-build
|
||||
depends_on:
|
||||
- image-build
|
||||
- image-build-a3
|
||||
key: upload-nightly-pipeline
|
||||
if: >-
|
||||
(build.branch == "main" && build.env("NIGHTLY") == "1") ||
|
||||
|
||||
@@ -22,3 +22,31 @@ steps:
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: nightly-tts-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
steps:
|
||||
- label: ":full_moon: TTS · Perf Test"
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-2"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpecPatch:
|
||||
imagePullSecrets:
|
||||
- name: swr-secret
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
@@ -6,6 +6,8 @@ ARG APP_DIR=/vllm-workspace/vllm-omni
|
||||
WORKDIR ${APP_DIR}
|
||||
COPY . .
|
||||
|
||||
RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
|
||||
@@ -6,6 +6,8 @@ ARG APP_DIR=/vllm-workspace/vllm-omni
|
||||
WORKDIR ${APP_DIR}
|
||||
COPY . .
|
||||
|
||||
RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user