[AMD][CI] Fix AMD CI (Partial) (#6704)
Signed-off-by: root <root@tw020.mia.tensorwave.lan> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: root <root@tw020.mia.tensorwave.lan>
This commit is contained in:
1 parent
c201fd77c6
commit
15c6ab8fa7
6 files changed
+196
-62
No files matched your search
@@ -1,4 +1,7 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
|
|
||||||
# vllm-omni customized version
|
# vllm-omni customized version
|
||||||
# Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/bootstrap-amd.sh
|
# Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/bootstrap-amd.sh
|
||||||
# Last synced: 2025-12-15
|
# Last synced: 2025-12-15
|
||||||
@@ -6,6 +9,9 @@
|
|||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
# The bootstrap runs from the repository root; ShellCheck does not follow
|
||||||
|
# sourced files unless invoked with -x.
|
||||||
|
# shellcheck disable=SC1091
|
||||||
source .buildkite/common/scripts/resolve_skip_ci.sh
|
source .buildkite/common/scripts/resolve_skip_ci.sh
|
||||||
|
|
||||||
if [[ -z "${RUN_ALL:-}" ]]; then
|
if [[ -z "${RUN_ALL:-}" ]]; then
|
||||||
@@ -63,6 +69,8 @@ upload_pipeline() {
|
|||||||
# Install minijinja
|
# Install minijinja
|
||||||
ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
|
ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
|
||||||
curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
|
curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
|
||||||
|
# Installed by the minijinja bootstrap above and only present on the CI agent.
|
||||||
|
# shellcheck disable=SC1091
|
||||||
source /var/lib/buildkite-agent/.cargo/env
|
source /var/lib/buildkite-agent/.cargo/env
|
||||||
|
|
||||||
if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
|
if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
|
||||||
@@ -90,22 +98,82 @@ upload_pipeline() {
|
|||||||
cd .buildkite/amd
|
cd .buildkite/amd
|
||||||
|
|
||||||
# Select test definition file: merge suite for main, ready suite for PRs.
|
# Select test definition file: merge suite for main, ready suite for PRs.
|
||||||
# For debugging, DEBUG_TEST_YAML can override the selection — accepts
|
# For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
|
||||||
# "merge" or "ready" (case-insensitive).
|
# "merge" and/or "ready" (case-insensitive). Multiple suites are combined
|
||||||
|
# before rendering so they share one amd-build step.
|
||||||
if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
|
if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
|
||||||
case "${DEBUG_TEST_YAML,,}" in
|
declare -a DEBUG_TEST_SPECS=()
|
||||||
merge)
|
declare -A SEEN_DEBUG_TESTS=()
|
||||||
TEST_YAML="test-amd-merge.yml"
|
IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
|
||||||
;;
|
|
||||||
ready)
|
for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
|
||||||
TEST_YAML="test-amd-ready.yml"
|
# Permit readable values such as "ready, merge," and ignore the
|
||||||
;;
|
# empty item produced by a trailing comma.
|
||||||
*)
|
requested_test="${requested_test//[[:space:]]/}"
|
||||||
echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
|
[[ -z "$requested_test" ]] && continue
|
||||||
|
|
||||||
|
if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
|
||||||
|
echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
fi
|
||||||
esac
|
SEEN_DEBUG_TESTS[$requested_test]=1
|
||||||
echo "DEBUG_TEST_YAML override: using $TEST_YAML"
|
|
||||||
|
case "$requested_test" in
|
||||||
|
ready)
|
||||||
|
DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
|
||||||
|
;;
|
||||||
|
merge)
|
||||||
|
DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
|
||||||
|
echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
|
||||||
|
exit 1
|
||||||
|
elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
|
||||||
|
TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
|
||||||
|
else
|
||||||
|
TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
|
||||||
|
python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
output_path, *suite_specs = sys.argv[1:]
|
||||||
|
combined = {"env": {}, "steps": []}
|
||||||
|
|
||||||
|
for suite_spec in suite_specs:
|
||||||
|
group_name, input_path = suite_spec.split(":", 1)
|
||||||
|
with open(input_path, encoding="utf-8") as test_file:
|
||||||
|
suite = yaml.safe_load(test_file)
|
||||||
|
|
||||||
|
for name, value in (suite.get("env") or {}).items():
|
||||||
|
previous = combined["env"].get(name, value)
|
||||||
|
if previous != value:
|
||||||
|
raise ValueError(
|
||||||
|
f"Conflicting environment value for {name}: {previous!r} != {value!r}"
|
||||||
|
)
|
||||||
|
combined["env"][name] = value
|
||||||
|
|
||||||
|
suite_steps = []
|
||||||
|
for entry in suite.get("steps") or []:
|
||||||
|
if "group" in entry:
|
||||||
|
suite_steps.extend(entry.get("steps") or [])
|
||||||
|
else:
|
||||||
|
suite_steps.append(entry)
|
||||||
|
combined["steps"].append({"group": group_name, "steps": suite_steps})
|
||||||
|
|
||||||
|
with open(output_path, "w", encoding="utf-8") as output_file:
|
||||||
|
yaml.safe_dump(combined, output_file, sort_keys=False)
|
||||||
|
PY
|
||||||
|
fi
|
||||||
|
echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
|
||||||
elif [[ $BUILDKITE_BRANCH == "main" ]]; then
|
elif [[ $BUILDKITE_BRANCH == "main" ]]; then
|
||||||
TEST_YAML="test-amd-merge.yml"
|
TEST_YAML="test-amd-merge.yml"
|
||||||
else
|
else
|
||||||
@@ -130,19 +198,20 @@ upload_pipeline() {
|
|||||||
> pipeline.yaml
|
> pipeline.yaml
|
||||||
)
|
)
|
||||||
cat pipeline.yaml
|
cat pipeline.yaml
|
||||||
|
if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
|
||||||
|
rm -f -- "$TEST_YAML"
|
||||||
|
fi
|
||||||
buildkite-agent artifact upload pipeline.yaml
|
buildkite-agent artifact upload pipeline.yaml
|
||||||
buildkite-agent pipeline upload pipeline.yaml
|
buildkite-agent pipeline upload pipeline.yaml
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
|
|
||||||
get_diff() {
|
get_diff() {
|
||||||
$(git add .)
|
git diff --name-only --diff-filter=ACMDR "$(git merge-base origin/main HEAD)"
|
||||||
echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get_diff_main() {
|
get_diff_main() {
|
||||||
$(git add .)
|
git diff --name-only --diff-filter=ACMDR HEAD~1
|
||||||
echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file_diff=$(get_diff)
|
file_diff=$(get_diff)
|
||||||
@@ -181,7 +250,7 @@ for file in $file_diff; do
|
|||||||
# First check if file matches any pattern
|
# First check if file matches any pattern
|
||||||
matches_pattern=0
|
matches_pattern=0
|
||||||
for pattern in "${patterns[@]}"; do
|
for pattern in "${patterns[@]}"; do
|
||||||
if [[ $file == $pattern* ]] || [[ $file == $pattern ]]; then
|
if [[ $file == "$pattern"* ]] || [[ $file == "$pattern" ]]; then
|
||||||
matches_pattern=1
|
matches_pattern=1
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
@@ -191,7 +260,7 @@ for file in $file_diff; do
|
|||||||
if [[ $matches_pattern -eq 1 ]]; then
|
if [[ $matches_pattern -eq 1 ]]; then
|
||||||
matches_ignore=0
|
matches_ignore=0
|
||||||
for ignore in "${ignore_patterns[@]}"; do
|
for ignore in "${ignore_patterns[@]}"; do
|
||||||
if [[ $file == $ignore* ]] || [[ $file == $ignore ]]; then
|
if [[ $file == "$ignore"* ]] || [[ $file == "$ignore" ]]; then
|
||||||
matches_ignore=1
|
matches_ignore=1
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ steps:
|
|||||||
|
|
||||||
- label: "Simple · Diffusion Test · Shard %N/%t"
|
- label: "Simple · Diffusion Test · Shard %N/%t"
|
||||||
agent_pool: mi300_1
|
agent_pool: mi300_1
|
||||||
parallelism: 2
|
parallelism: 4
|
||||||
depends_on: amd-build
|
depends_on: amd-build
|
||||||
mirror_hardwares: [amdproduction]
|
mirror_hardwares: [amdproduction]
|
||||||
grade: Blocking
|
grade: Blocking
|
||||||
@@ -71,26 +71,30 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- |
|
- pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
|
||||||
bash -c '
|
- pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
|
||||||
set +e
|
|
||||||
pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
|
|
||||||
EXIT1=\$?
|
|
||||||
pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
|
|
||||||
EXIT2=\$?
|
|
||||||
exit \$((EXIT1 | EXIT2))
|
|
||||||
'
|
|
||||||
|
|
||||||
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
|
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 1/2"
|
||||||
agent_pool: mi300_4
|
agent_pool: mi300_4
|
||||||
depends_on: amd-build
|
depends_on: amd-build
|
||||||
mirror_hardwares: [amdproduction]
|
mirror_hardwares: [amdproduction]
|
||||||
|
timeout_in_minutes: 40
|
||||||
grade: Blocking
|
grade: Blocking
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
|
- pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
|
||||||
- timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
|
|
||||||
|
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 2/2"
|
||||||
|
agent_pool: mi300_4
|
||||||
|
depends_on: amd-build
|
||||||
|
mirror_hardwares: [amdproduction]
|
||||||
|
timeout_in_minutes: 40
|
||||||
|
grade: Blocking
|
||||||
|
commands:
|
||||||
|
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
|
||||||
|
|
||||||
# merge-only tests
|
# merge-only tests
|
||||||
- label: "Diffusion Tensor Parallelism Test"
|
- label: "Diffusion Tensor Parallelism Test"
|
||||||
@@ -175,17 +179,41 @@ steps:
|
|||||||
|
|
||||||
- label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t"
|
- label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t"
|
||||||
agent_pool: mi300_1
|
agent_pool: mi300_1
|
||||||
parallelism: 2
|
parallelism: 3
|
||||||
depends_on: amd-build
|
depends_on: amd-build
|
||||||
mirror_hardwares: [amdproduction]
|
mirror_hardwares: [amdproduction]
|
||||||
grade: Blocking
|
grade: Blocking
|
||||||
|
timeout_in_minutes: 80
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
bash -c '
|
timeout --signal=TERM --kill-after=2m 80m bash -c '
|
||||||
export VLLM_LOGGING_LEVEL=DEBUG
|
export VLLM_LOGGING_LEVEL=DEBUG
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
TESTS=(
|
||||||
|
"tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_default_cuda_graph_startup[async_chunk]"
|
||||||
|
"tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_002[async_chunk]"
|
||||||
|
"tests/e2e/offline_inference/test_qwen3_tts_customvoice.py::test_text_to_audio_001[no_cuda_graph]"
|
||||||
|
)
|
||||||
|
TEST="$${TESTS[$$BUILDKITE_PARALLEL_JOB]}"
|
||||||
|
pytest -s -v "$$TEST" -m "advanced_model" --run-level "advanced_model"
|
||||||
|
'
|
||||||
|
|
||||||
|
# TODO: Restore this case to the blocking sharded step once its long-running
|
||||||
|
# Whisper validation on AMD CI is resolved.
|
||||||
|
- label: "Qwen3-TTS CustomVoice async_chunk Test (quarantined)"
|
||||||
|
agent_pool: mi300_1
|
||||||
|
depends_on: amd-build
|
||||||
|
mirror_hardwares: [amdproduction]
|
||||||
|
grade: NonBlocking
|
||||||
|
timeout_in_minutes: 80
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
timeout --signal=TERM --kill-after=2m 75m bash -c '
|
||||||
|
export VLLM_LOGGING_LEVEL=DEBUG
|
||||||
|
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||||
|
pytest -s -v "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_001[async_chunk]" -m "advanced_model" --run-level "advanced_model"
|
||||||
'
|
'
|
||||||
|
|
||||||
- label: "Qwen3-TTS Base E2E Test"
|
- label: "Qwen3-TTS Base E2E Test"
|
||||||
|
|||||||
@@ -191,10 +191,7 @@ steps:
|
|||||||
mirror_hardwares: [amdproduction]
|
mirror_hardwares: [amdproduction]
|
||||||
grade: Blocking
|
grade: Blocking
|
||||||
commands:
|
commands:
|
||||||
- |
|
- pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
|
||||||
timeout 45m bash -c '
|
|
||||||
pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
|
|
||||||
'
|
|
||||||
|
|
||||||
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
||||||
# Need to debug before reneable numerical changes across large PRs
|
# Need to debug before reneable numerical changes across large PRs
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Uses ``Tongyi-MAI/Z-Image-Turbo`` as the parity model.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import gc
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -75,8 +76,13 @@ def _run_generate(
|
|||||||
if num_requests < 2:
|
if num_requests < 2:
|
||||||
raise ValueError("num_requests must be >= 2 (1 warmup + >=1 timed)")
|
raise ValueError("num_requests must be >= 2 (1 warmup + >=1 timed)")
|
||||||
|
|
||||||
|
gc.collect()
|
||||||
current_omni_platform.empty_cache()
|
current_omni_platform.empty_cache()
|
||||||
device_index = current_omni_platform.current_device()
|
device_index = current_omni_platform.current_device()
|
||||||
|
with current_omni_platform.device(device_index):
|
||||||
|
free_bytes, total_bytes = current_omni_platform.mem_get_info()
|
||||||
|
initial_used_mb = (total_bytes - free_bytes) / (1024**2)
|
||||||
|
|
||||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||||
monitor.start()
|
monitor.start()
|
||||||
try:
|
try:
|
||||||
@@ -113,7 +119,10 @@ def _run_generate(
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
median_time_s = float(np.median(per_request_times_s))
|
median_time_s = float(np.median(per_request_times_s))
|
||||||
peak_memory_mb = monitor.peak_used_mb
|
# DeviceMemoryMonitor samples absolute device usage. Normalize by
|
||||||
|
# this run's starting usage so compiler and allocator caches retained
|
||||||
|
# by the preceding TP=1 run are not charged to the TP=2 run.
|
||||||
|
peak_memory_mb = max(0.0, monitor.peak_used_mb - initial_used_mb)
|
||||||
return _extract_single_image([last_output]), median_time_s, peak_memory_mb
|
return _extract_single_image([last_output]), median_time_s, peak_memory_mb
|
||||||
finally:
|
finally:
|
||||||
monitor.stop()
|
monitor.stop()
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
|
|
||||||
|
import gc
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -15,7 +16,11 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
|||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
AUDIO_MODEL: dict[str, dict[str, int | None]] = {
|
AUDIO_MODEL: dict[str, dict[str, int | None]] = {
|
||||||
"stabilityai/stable-audio-open-1.0": {"cuda": 1500, "rocm": 1500},
|
# The inference peak includes backend workspaces as well as model weights.
|
||||||
|
# On ROCm, large MIOpen workspaces and allocator fragmentation can mask the
|
||||||
|
# resident-weight reduction, so use a conservative floor that still catches
|
||||||
|
# a disabled/no-op layerwise offloader.
|
||||||
|
"stabilityai/stable-audio-open-1.0": {"cuda": 1500, "rocm": 512},
|
||||||
}
|
}
|
||||||
|
|
||||||
IMAGE_VIDEO_MODELS: dict[str, dict[str, int | None]] = {
|
IMAGE_VIDEO_MODELS: dict[str, dict[str, int | None]] = {
|
||||||
@@ -63,8 +68,9 @@ def run_inference(
|
|||||||
) -> tuple[float, Any]:
|
) -> tuple[float, Any]:
|
||||||
current_omni_platform.empty_cache()
|
current_omni_platform.empty_cache()
|
||||||
device_index = current_omni_platform.current_device()
|
device_index = current_omni_platform.current_device()
|
||||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
with current_omni_platform.device(device_index):
|
||||||
monitor.start()
|
free_bytes, total_bytes = current_omni_platform.mem_get_info()
|
||||||
|
initial_used_mb = (total_bytes - free_bytes) / (1024**2)
|
||||||
|
|
||||||
if model_name in AUDIO_MODEL:
|
if model_name in AUDIO_MODEL:
|
||||||
params = AUDIO_MODEL_PARAMS
|
params = AUDIO_MODEL_PARAMS
|
||||||
@@ -78,22 +84,37 @@ def run_inference(
|
|||||||
# cache_backend="cache_dit",
|
# cache_backend="cache_dit",
|
||||||
**params["runner_params"],
|
**params["runner_params"],
|
||||||
) as runner:
|
) as runner:
|
||||||
|
# Measure steady-state inference memory, not model construction. Enabling
|
||||||
|
# layerwise offload first loads the model and then replaces each block's
|
||||||
|
# device storage with CPU-backed weights. Monitoring that transition
|
||||||
|
# captures both the original model and temporary staging allocations,
|
||||||
|
# which is not representative of layerwise-offloaded inference.
|
||||||
|
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||||
current_omni_platform.reset_peak_memory_stats()
|
current_omni_platform.reset_peak_memory_stats()
|
||||||
|
monitor.start()
|
||||||
|
|
||||||
# Refer to tests/e2e/offline_inference/test_wan22.py
|
try:
|
||||||
# Use minimal settings for testing
|
# Refer to tests/e2e/offline_inference/test_wan22.py
|
||||||
output = runner.omni.generate(
|
# Use minimal settings for testing
|
||||||
"A cat sitting on a table",
|
output = runner.omni.generate(
|
||||||
OmniDiffusionSamplingParams(
|
"A cat sitting on a table",
|
||||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
OmniDiffusionSamplingParams(
|
||||||
guidance_scale=1.0,
|
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||||
num_inference_steps=num_inference_steps,
|
guidance_scale=1.0,
|
||||||
**params["sampler_params"],
|
num_inference_steps=num_inference_steps,
|
||||||
),
|
**params["sampler_params"],
|
||||||
)
|
),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
monitor.stop()
|
||||||
|
|
||||||
peak = monitor.peak_used_mb
|
# DeviceMemoryMonitor reports absolute device usage. Subtract this run's
|
||||||
monitor.stop()
|
# starting usage so process-wide compiler/workspace caches retained from a
|
||||||
|
# previous run do not make the second measurement order-dependent.
|
||||||
|
peak = max(0.0, monitor.peak_used_mb - initial_used_mb)
|
||||||
|
|
||||||
|
gc.collect()
|
||||||
|
current_omni_platform.empty_cache()
|
||||||
|
|
||||||
return peak, output
|
return peak, output
|
||||||
|
|
||||||
@@ -148,8 +169,10 @@ def test_layerwise_offload_diffusion_model(model_name: str):
|
|||||||
assert expected_saved_memory is not None
|
assert expected_saved_memory is not None
|
||||||
|
|
||||||
# Verify that layerwise offloading significantly reduces memory usage
|
# Verify that layerwise offloading significantly reduces memory usage
|
||||||
# Passes only if the actual savings exceeds the expected savings
|
# Passes only if the actual savings meets the expected savings
|
||||||
assert layerwise_offload_peak_memory + expected_saved_memory < no_offload_peak_memory, (
|
actual_saved_memory = no_offload_peak_memory - layerwise_offload_peak_memory
|
||||||
|
assert layerwise_offload_peak_memory + expected_saved_memory <= no_offload_peak_memory, (
|
||||||
f"Layerwise offload peak memory {layerwise_offload_peak_memory} MB "
|
f"Layerwise offload peak memory {layerwise_offload_peak_memory} MB "
|
||||||
f"should be significantly less than no offload peak memory {no_offload_peak_memory} MB"
|
f"should be at least {expected_saved_memory} MB less than no offload peak memory "
|
||||||
|
f"{no_offload_peak_memory} MB (actual savings: {actual_saved_memory} MB)"
|
||||||
)
|
)
|
||||||
+10
-2
@@ -150,8 +150,16 @@ def _print_device_processes() -> None:
|
|||||||
elif current_omni_platform.is_npu():
|
elif current_omni_platform.is_npu():
|
||||||
_run_smi("Ascend NPU Information (npu-smi info)", ["npu-smi", "info"], 40)
|
_run_smi("Ascend NPU Information (npu-smi info)", ["npu-smi", "info"], 40)
|
||||||
elif current_omni_platform.is_rocm():
|
elif current_omni_platform.is_rocm():
|
||||||
_run_smi("AMD GPU Information (amd-smi)", ["amd-smi"], 30)
|
# amd-smi can enter uninterruptible sleep in the amdgpu CPER ioctl
|
||||||
_run_smi("Detailed AMD GPU Processes (amd-smi process)", ["amd-smi", "process"], 100, timeout=3)
|
# (amdgpu_cper_ring_write). In that state subprocess.run's timeout
|
||||||
|
# cannot reap it, causing this diagnostic hook to hang the entire test.
|
||||||
|
# rocm-smi provides the information needed here without that ioctl.
|
||||||
|
_run_smi(
|
||||||
|
"AMD GPU Information (rocm-smi)",
|
||||||
|
["rocm-smi", "--showuse", "--showmeminfo", "vram"],
|
||||||
|
60,
|
||||||
|
)
|
||||||
|
_run_smi("Detailed AMD GPU Processes (rocm-smi)", ["rocm-smi", "--showpids"], 100, timeout=3)
|
||||||
elif current_omni_platform.is_xpu():
|
elif current_omni_platform.is_xpu():
|
||||||
_run_smi("Intel XPU Information (xpu-smi discovery)", ["xpu-smi", "discovery"], 40)
|
_run_smi("Intel XPU Information (xpu-smi discovery)", ["xpu-smi", "discovery"], 40)
|
||||||
elif current_omni_platform.is_musa():
|
elif current_omni_platform.is_musa():
|
||||||
|
|||||||
Reference in new issue
Block a user