Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Zhou Taichang
|
a9012a1b79 |
No files matched your search
@@ -1,103 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Run vLLM-Omni ROCm tests directly in the MI300 Kubernetes pod. The pod's
|
||||
# container image is selected by test-template-amd-omni.j2; MI300 has no DinD.
|
||||
set -euo pipefail
|
||||
|
||||
: "${PYTHONFAULTHANDLER:=1}"
|
||||
: "${HF_HOME:=/home/buildkite-agent/huggingface}"
|
||||
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
|
||||
: "${HF_HUB_ETAG_TIMEOUT:=60}"
|
||||
: "${MIOPEN_DEBUG_CONV_DIRECT:=0}"
|
||||
: "${MIOPEN_DEBUG_CONV_GEMM:=0}"
|
||||
: "${VLLM_ROCM_USE_AITER:=0}"
|
||||
export PYTHONFAULTHANDLER HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
|
||||
export MIOPEN_DEBUG_CONV_DIRECT MIOPEN_DEBUG_CONV_GEMM VLLM_ROCM_USE_AITER
|
||||
export PYTORCH_ROCM_ARCH=""
|
||||
export PYTHONPATH="${PYTHONPATH:-..}"
|
||||
|
||||
if [[ "${VLLM_CI_DOCKER_DISABLED:-0}" != "1" ]]; then
|
||||
echo "Error: MI300 CI must run natively with Docker disabled." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -n "${TEST_COMMANDS:-}" ]]; then
|
||||
commands="${TEST_COMMANDS}"
|
||||
else
|
||||
commands="$*"
|
||||
fi
|
||||
if [[ -z "${commands}" ]]; then
|
||||
echo "Error: No test commands provided." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "--- Native in-pod ROCm CI"
|
||||
|
||||
job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
|
||||
job_id="${job_id//[^A-Za-z0-9_.-]/_}"
|
||||
native_cache_root="/tmp/vllm-omni-native-${job_id}"
|
||||
TMPDIR="${native_cache_root}/tmp"
|
||||
TORCHINDUCTOR_CACHE_DIR="${native_cache_root}/torchinductor"
|
||||
TRITON_CACHE_DIR="${native_cache_root}/triton"
|
||||
VLLM_CACHE_ROOT="${native_cache_root}/vllm"
|
||||
XDG_CACHE_HOME="${native_cache_root}/xdg"
|
||||
HF_DATASETS_CACHE="${native_cache_root}/huggingface/datasets"
|
||||
export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT
|
||||
export XDG_CACHE_HOME HF_DATASETS_CACHE
|
||||
mkdir -p "${TMPDIR}" "${TORCHINDUCTOR_CACHE_DIR}" "${TRITON_CACHE_DIR}" \
|
||||
"${VLLM_CACHE_ROOT}" "${XDG_CACHE_HOME}" "${HF_HOME}" \
|
||||
"${HF_DATASETS_CACHE}"
|
||||
|
||||
if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
|
||||
if ! command -v findmnt >/dev/null 2>&1; then
|
||||
echo "Error: findmnt is required to verify the Hugging Face cache mount." >&2
|
||||
exit 1
|
||||
fi
|
||||
hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
|
||||
if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
|
||||
echo "Error: MI300 CI requires a persistent volume at or above ${HF_HOME}." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
rocminfo
|
||||
|
||||
expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
expected = int(os.environ.get("VLLM_CI_EXPECTED_GPU_COUNT", "1"))
|
||||
assert torch.version.hip, "PyTorch is not a ROCm build"
|
||||
assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
|
||||
actual = torch.cuda.device_count()
|
||||
assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
|
||||
PY
|
||||
|
||||
echo "Commands:${commands}"
|
||||
if /bin/bash -o pipefail -c '
|
||||
set -E
|
||||
test_status=0
|
||||
trap '\''
|
||||
command_status=$?
|
||||
# Keep running subsequent commands, but retain a failure for the final exit.
|
||||
# Prefer a test failure over pytest "no tests collected" when both occur.
|
||||
if (( test_status == 0 || (test_status == 5 && command_status != 5) )); then
|
||||
test_status=${command_status}
|
||||
fi
|
||||
'\'' ERR
|
||||
eval "$1"
|
||||
exit "${test_status}"
|
||||
' _ "${commands}"; then
|
||||
exit 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [[ ${exit_code} -eq 5 && "${VLLM_CI_ALLOW_NO_TESTS:-0}" == "1" ]]; then
|
||||
echo "Pytest collected no tests; VLLM_CI_ALLOW_NO_TESTS=1, treating exit code 5 as success."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exit "${exit_code}"
|
||||
@@ -1,284 +0,0 @@
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
steps:
|
||||
|
||||
- group: ":card_index_dividers: Simple Test"
|
||||
steps:
|
||||
- label: "Simple · Model Executor Test · Shard %N/%t"
|
||||
agent_pool: mi300_1
|
||||
parallelism: 2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
|
||||
|
||||
- label: "Simple · Diffusion Test · Shard %N/%t"
|
||||
agent_pool: mi300_1
|
||||
parallelism: 4
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
|
||||
|
||||
- label: "Simple · Engine&Entrypoints Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
- "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
|
||||
|
||||
- label: "Simple · Other Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py"
|
||||
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
steps:
|
||||
- label: "Diffusion · LoRA Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
artifact_paths:
|
||||
- lora-server.log
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" 2>&1 | tee lora-server.log
|
||||
|
||||
- label: "Diffusion Model CPU offloading Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
|
||||
- >-
|
||||
pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
|
||||
--deselect='tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
|
||||
|
||||
# TODO: Diagnose unstable Stable Audio layerwise-offload memory savings on ROCm.
|
||||
- label: "Diffusion Stable Audio Layerwise Offload Test (quarantined)"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: NonBlocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- >-
|
||||
pytest -s -v
|
||||
'tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
|
||||
|
||||
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 1/2"
|
||||
agent_pool: mi300_4
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
timeout_in_minutes: 40
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
|
||||
|
||||
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 2/2"
|
||||
agent_pool: mi300_4
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
timeout_in_minutes: 40
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
|
||||
|
||||
# merge-only tests
|
||||
- label: "Diffusion Tensor Parallelism Test"
|
||||
agent_pool: mi300_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
|
||||
|
||||
- label: "Diffusion Model Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
|
||||
|
||||
- label: "Diffusion GPU Worker Test"
|
||||
agent_pool: mi300_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
|
||||
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
|
||||
## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
|
||||
# - label: "Audio Generation Model Test"
|
||||
# agent_pool: mi300_1
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
|
||||
|
||||
- label: "Engine Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py
|
||||
|
||||
# This is from test-nightly.yml.
|
||||
# This model seems to be important, so I am trying to run it here.
|
||||
# Since this is from test-nightly.yml,
|
||||
# I keep this non-blocking.
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
steps:
|
||||
- label: "Omni Model Test Qwen2-5-Omni"
|
||||
agent_pool: mi300_4
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
|
||||
|
||||
- label: "Omni Model Test Qwen3-Omni"
|
||||
agent_pool: mi300_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
|
||||
- label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t"
|
||||
agent_pool: mi300_1
|
||||
parallelism: 3
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
timeout_in_minutes: 80
|
||||
commands:
|
||||
- |
|
||||
timeout --signal=TERM --kill-after=2m 80m bash -c '
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
TESTS=(
|
||||
"tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_default_cuda_graph_startup[async_chunk]"
|
||||
"tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_002[async_chunk]"
|
||||
"tests/e2e/offline_inference/test_qwen3_tts_customvoice.py::test_text_to_audio_001[no_cuda_graph]"
|
||||
)
|
||||
TEST="$${TESTS[$$BUILDKITE_PARALLEL_JOB]}"
|
||||
pytest -s -v "$$TEST" -m "advanced_model" --run-level "advanced_model"
|
||||
'
|
||||
|
||||
# TODO: Restore this case to the blocking sharded step once its long-running
|
||||
# Whisper validation on AMD CI is resolved.
|
||||
- label: "Qwen3-TTS CustomVoice async_chunk Test (quarantined)"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: NonBlocking
|
||||
timeout_in_minutes: 80
|
||||
commands:
|
||||
- |
|
||||
timeout --signal=TERM --kill-after=2m 75m bash -c '
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
pytest -s -v "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_001[async_chunk]" -m "advanced_model" --run-level "advanced_model"
|
||||
'
|
||||
|
||||
- label: "Qwen3-TTS Base E2E Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- timeout 40m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model and cuda" --run-level "advanced_model"
|
||||
|
||||
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
||||
# Need to debug before reneable numerical changes across large PRs
|
||||
# # split Bagel Model Test with H100 (Real Weights) into three tests
|
||||
# - label: "Bagel Text2Img Model Test (1/3)"
|
||||
# agent_pool: mi300_1
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
# - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
|
||||
|
||||
# - label: "Bagel Img2Img Model Test (2/3)"
|
||||
# agent_pool: mi300_1
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
# - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
|
||||
|
||||
# - label: "Bagel Online Serving Test (3/3)"
|
||||
# agent_pool: mi300_1
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
# - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
|
||||
@@ -1,233 +0,0 @@
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
steps:
|
||||
|
||||
- group: ":card_index_dividers: Simple Test"
|
||||
steps:
|
||||
- label: "Simple · Model Executor Test · Shard %N/%t"
|
||||
agent_pool: mi300_1
|
||||
parallelism: 2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
|
||||
|
||||
- label: "Simple · Diffusion Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py"
|
||||
|
||||
- label: "Simple · Engine&Entrypoints Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
- "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
|
||||
|
||||
- label: "Simple · Other Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py"
|
||||
|
||||
- label: "Custom Pipeline Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
|
||||
|
||||
|
||||
- label: "Voxtral TTS CUDA Unit Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
|
||||
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
steps:
|
||||
- label: "Diffusion · Batch Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
|
||||
|
||||
- label: "Diffusion · Offloader Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
|
||||
|
||||
- label: "Diffusion · Cache Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
|
||||
|
||||
- label: "Diffusion Sequence Parallelism Test"
|
||||
agent_pool: mi300_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
|
||||
|
||||
- label: "Diffusion GPU Worker Test"
|
||||
agent_pool: mi300_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
|
||||
|
||||
- label: "Diffusion · Model Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
|
||||
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
|
||||
## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
|
||||
# - label: "Audio Generation Model Test"
|
||||
# agent_pool: mi300_1
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
|
||||
|
||||
- label: "Engine Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- |
|
||||
timeout 15m bash -c '
|
||||
pytest -s -v tests/engine/test_async_omni_engine_abort.py
|
||||
'
|
||||
|
||||
# - label: "Omni Model Test Qwen3-Omni"
|
||||
# agent_pool: mi300_2
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
# - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
|
||||
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
steps:
|
||||
- label: "Qwen3-TTS E2E Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
|
||||
|
||||
- label: "CosyVoice3-TTS E2E Test"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
|
||||
|
||||
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
||||
# Need to debug before reneable numerical changes across large PRs
|
||||
# - label: "Bagel Text2Img Model Test"
|
||||
# agent_pool: mi300_1
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
# - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
|
||||
|
||||
# - label: "Bagel Img2Img Model Test"
|
||||
# agent_pool: mi300_1
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
# - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
|
||||
|
||||
# - label: "Bagel Online Serving Test"
|
||||
# agent_pool: mi300_1
|
||||
# depends_on: amd-build
|
||||
# mirror_hardwares: [amdproduction]
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
# - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
|
||||
@@ -1,237 +0,0 @@
|
||||
{# vllm-omni customized version
|
||||
Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
|
||||
Last synced: 2026-08-04
|
||||
Modifications: Use the shared MI300 queues and Omni command-passing contract
|
||||
#}
|
||||
{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
|
||||
{% set default_working_dir = "/app/vllm-omni" %}
|
||||
|
||||
{# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
|
||||
{% macro amd_native_k8s_plugin(indent, step) -%}
|
||||
{{ indent }}plugins:
|
||||
{{ indent }} - kubernetes:
|
||||
{{ indent }} podSpecPatch:
|
||||
{{ indent }} automountServiceAccountToken: false
|
||||
{{ indent }} securityContext:
|
||||
{{ indent }} seccompProfile:
|
||||
{{ indent }} type: RuntimeDefault
|
||||
{{ indent }} imagePullSecrets:
|
||||
{{ indent }} - name: docker-config
|
||||
{{ indent }} containers:
|
||||
{{ indent }} - name: container-0
|
||||
{{ indent }} image: {{ docker_image_amd }}
|
||||
{{ indent }} imagePullPolicy: Always
|
||||
{{ indent }} securityContext:
|
||||
{{ indent }} allowPrivilegeEscalation: false
|
||||
{{ indent }} capabilities:
|
||||
{{ indent }} add:
|
||||
{{ indent }} - IPC_LOCK
|
||||
{{ indent }} resources:
|
||||
{{ indent }} limits:
|
||||
{{ indent }} amd.com/gpu: "{{ step.agent_pool[-1:] }}"
|
||||
{{ indent }} requests:
|
||||
{{ indent }} amd.com/gpu: "{{ step.agent_pool[-1:] }}"
|
||||
{{ indent }} volumeMounts:
|
||||
{{ indent }} - name: devshm
|
||||
{{ indent }} mountPath: /dev/shm
|
||||
{{ indent }} env:
|
||||
{{ indent }} - name: AMD_CI_RUNTIME
|
||||
{{ indent }} value: native
|
||||
{{ indent }} - name: NATIVE_CI
|
||||
{{ indent }} value: "true"
|
||||
{{ indent }} - name: VLLM_CI_DOCKER_DISABLED
|
||||
{{ indent }} value: "1"
|
||||
{{ indent }} - name: VLLM_CI_EXPECTED_GPU_COUNT
|
||||
{{ indent }} value: "{{ step.agent_pool[-1:] }}"
|
||||
{{ indent }} - name: HF_HOME
|
||||
{{ indent }} value: /home/buildkite-agent/huggingface
|
||||
{{ indent }} - name: MIOPEN_DEBUG_CONV_DIRECT
|
||||
{{ indent }} value: "0"
|
||||
{{ indent }} - name: MIOPEN_DEBUG_CONV_GEMM
|
||||
{{ indent }} value: "0"
|
||||
{{ indent }} - name: VLLM_ROCM_USE_AITER
|
||||
{{ indent }} value: "0"
|
||||
{{ indent }} - name: PYTORCH_ROCM_ARCH
|
||||
{{ indent }} value: ""
|
||||
{{ indent }} volumes:
|
||||
{{ indent }} - name: devshm
|
||||
{{ indent }} emptyDir:
|
||||
{{ indent }} medium: Memory
|
||||
{{ indent }} sizeLimit: 16Gi
|
||||
{%- endmacro %}
|
||||
|
||||
{% if env %}
|
||||
env:
|
||||
{% for name in env %}
|
||||
{{ name }}: "{{ env[name] }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
steps:
|
||||
- group: "AMD Tests"
|
||||
depends_on: ~
|
||||
steps:
|
||||
- label: "AMD: :docker: build image"
|
||||
depends_on: ~
|
||||
soft_fail: false
|
||||
commands:
|
||||
- "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
|
||||
- "docker push {{ docker_image_amd }}"
|
||||
key: "amd-build"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: -1 # Agent was lost
|
||||
limit: 2
|
||||
- exit_status: -10 # Agent was lost
|
||||
limit: 2
|
||||
- exit_status: 128 # Git connectivity issues
|
||||
limit: 2
|
||||
- exit_status: 1 # Machine occasionally fail
|
||||
limit: 1
|
||||
agents:
|
||||
queue: amd-cpu
|
||||
|
||||
{% for step_group in steps %}
|
||||
{% if step_group.group %}
|
||||
- group: "{{ step_group.group }}"
|
||||
{% if step_group.depends_on %}
|
||||
depends_on: {{ step_group.depends_on }}
|
||||
{% endif %}
|
||||
steps:
|
||||
{% for step in step_group.steps %}
|
||||
{% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
|
||||
- label: "{{ step.agent_pool }}: {{ step.label }}"
|
||||
depends_on: amd-build
|
||||
agents:
|
||||
{% if step.agent_pool %}
|
||||
queue: amd_{{ step.agent_pool }}
|
||||
{% else %}
|
||||
queue: amd_mi300_1
|
||||
{% endif %}
|
||||
{{ amd_native_k8s_plugin(" ", step) }}
|
||||
{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
|
||||
{% set indented_cmd = cmd_body | replace("\n", "\n ") %}
|
||||
command: bash .buildkite/amd/scripts/run-amd-test.sh
|
||||
env:
|
||||
AMD_CI_RUNTIME: "native"
|
||||
NATIVE_CI: "true"
|
||||
VLLM_CI_DOCKER_DISABLED: "1"
|
||||
VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
|
||||
VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
|
||||
HF_HOME: "/home/buildkite-agent/huggingface"
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: "300"
|
||||
HF_HUB_ETAG_TIMEOUT: "60"
|
||||
MIOPEN_DEBUG_CONV_DIRECT: "0"
|
||||
MIOPEN_DEBUG_CONV_GEMM: "0"
|
||||
VLLM_ROCM_USE_AITER: "0"
|
||||
PYTORCH_ROCM_ARCH: ""
|
||||
{% if step.env %}
|
||||
{% for name in step.env %}
|
||||
{{ name }}: "{{ step.env[name] }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
TEST_COMMANDS: |-
|
||||
(command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
|
||||
{% if env %}
|
||||
{% for name in env %}
|
||||
export {{ name }}="{{ env[name] }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if step.env %}
|
||||
{% for name in step.env %}
|
||||
export {{ name }}="{{ step.env[name] }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if "mi250" in step.agent_pool %}
|
||||
python3 -m pip uninstall -y amd-aiter
|
||||
{% endif %}
|
||||
{{ indented_cmd | safe }}
|
||||
priority: 100
|
||||
{% if step.parallelism %}
|
||||
parallelism: {{ step.parallelism }}
|
||||
{% endif %}
|
||||
{% if step.artifact_paths %}
|
||||
artifact_paths:
|
||||
{% for path in step.artifact_paths %}
|
||||
- "{{ path }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if step.timeout_in_minutes %}
|
||||
timeout_in_minutes: {{ step.timeout_in_minutes }}
|
||||
{% endif %}
|
||||
{% if step.grade and step.grade == "Blocking" %}
|
||||
soft_fail: false
|
||||
{% else %}
|
||||
soft_fail: true
|
||||
{% endif%}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% elif step_group.mirror_hardwares and mirror_hw in step_group.mirror_hardwares %}
|
||||
{% set step = step_group %}
|
||||
- label: "{{ step.agent_pool }}: {{ step.label }}"
|
||||
depends_on: amd-build
|
||||
agents:
|
||||
{% if step.agent_pool %}
|
||||
queue: amd_{{ step.agent_pool }}
|
||||
{% else %}
|
||||
queue: amd_mi300_1
|
||||
{% endif %}
|
||||
{{ amd_native_k8s_plugin(" ", step) }}
|
||||
{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
|
||||
{% set indented_cmd = cmd_body | replace("\n", "\n ") %}
|
||||
command: bash .buildkite/amd/scripts/run-amd-test.sh
|
||||
env:
|
||||
AMD_CI_RUNTIME: "native"
|
||||
NATIVE_CI: "true"
|
||||
VLLM_CI_DOCKER_DISABLED: "1"
|
||||
VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
|
||||
VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
|
||||
HF_HOME: "/home/buildkite-agent/huggingface"
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: "300"
|
||||
HF_HUB_ETAG_TIMEOUT: "60"
|
||||
MIOPEN_DEBUG_CONV_DIRECT: "0"
|
||||
MIOPEN_DEBUG_CONV_GEMM: "0"
|
||||
VLLM_ROCM_USE_AITER: "0"
|
||||
PYTORCH_ROCM_ARCH: ""
|
||||
{% if step.env %}
|
||||
{% for name in step.env %}
|
||||
{{ name }}: "{{ step.env[name] }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
TEST_COMMANDS: |-
|
||||
(command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
|
||||
{% if env %}
|
||||
{% for name in env %}
|
||||
export {{ name }}="{{ env[name] }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if step.env %}
|
||||
{% for name in step.env %}
|
||||
export {{ name }}="{{ step.env[name] }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if "mi250" in step.agent_pool %}
|
||||
python3 -m pip uninstall -y amd-aiter
|
||||
{% endif %}
|
||||
{{ indented_cmd | safe }}
|
||||
priority: 100
|
||||
{% if step.parallelism %}
|
||||
parallelism: {{ step.parallelism }}
|
||||
{% endif %}
|
||||
{% if step.artifact_paths %}
|
||||
artifact_paths:
|
||||
{% for path in step.artifact_paths %}
|
||||
- "{{ path }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if step.timeout_in_minutes %}
|
||||
timeout_in_minutes: {{ step.timeout_in_minutes }}
|
||||
{% endif %}
|
||||
{% if step.grade and step.grade == "Blocking" %}
|
||||
soft_fail: false
|
||||
{% else %}
|
||||
soft_fail: true
|
||||
{% endif%}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
@@ -1,7 +1,4 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# vllm-omni customized version
|
||||
# Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/bootstrap-amd.sh
|
||||
# Last synced: 2025-12-15
|
||||
@@ -9,11 +6,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# The bootstrap runs from the repository root; ShellCheck does not follow
|
||||
# sourced files unless invoked with -x.
|
||||
# shellcheck disable=SC1091
|
||||
source .buildkite/common/scripts/resolve_skip_ci.sh
|
||||
|
||||
if [[ -z "${RUN_ALL:-}" ]]; then
|
||||
RUN_ALL=0
|
||||
fi
|
||||
@@ -30,6 +22,10 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
|
||||
AMD_MIRROR_HW="amdproduction"
|
||||
fi
|
||||
|
||||
if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
|
||||
DOCS_ONLY_DISABLE=0
|
||||
fi
|
||||
|
||||
fail_fast() {
|
||||
DISABLE_LABEL="ci-no-fail-fast"
|
||||
# If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
|
||||
@@ -69,8 +65,6 @@ upload_pipeline() {
|
||||
# Install minijinja
|
||||
ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
|
||||
curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
|
||||
# Installed by the minijinja bootstrap above and only present on the CI agent.
|
||||
# shellcheck disable=SC1091
|
||||
source /var/lib/buildkite-agent/.cargo/env
|
||||
|
||||
if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
|
||||
@@ -78,14 +72,14 @@ upload_pipeline() {
|
||||
fi
|
||||
|
||||
# Use local template file for vllm-omni
|
||||
cp .buildkite/amd/test-template-amd-omni.j2 .buildkite/amd/test-template.j2
|
||||
cp .buildkite/test-template-amd-omni.j2 .buildkite/test-template.j2
|
||||
|
||||
|
||||
# (WIP) Use pipeline generator instead of jinja template
|
||||
if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then
|
||||
if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then
|
||||
python -m pip install click pydantic
|
||||
python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
|
||||
buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml
|
||||
python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
|
||||
buildkite-agent pipeline upload .buildkite/pipeline.yaml
|
||||
exit 0
|
||||
fi
|
||||
echo "List file diff: $LIST_FILE_DIFF"
|
||||
@@ -95,95 +89,11 @@ upload_pipeline() {
|
||||
|
||||
FAIL_FAST=$(fail_fast)
|
||||
|
||||
cd .buildkite/amd
|
||||
|
||||
# Select test definition file: merge suite for main, ready suite for PRs.
|
||||
# For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
|
||||
# "merge" and/or "ready" (case-insensitive). Multiple suites are combined
|
||||
# before rendering so they share one amd-build step.
|
||||
if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
|
||||
declare -a DEBUG_TEST_SPECS=()
|
||||
declare -A SEEN_DEBUG_TESTS=()
|
||||
IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
|
||||
|
||||
for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
|
||||
# Permit readable values such as "ready, merge," and ignore the
|
||||
# empty item produced by a trailing comma.
|
||||
requested_test="${requested_test//[[:space:]]/}"
|
||||
[[ -z "$requested_test" ]] && continue
|
||||
|
||||
if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
|
||||
echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
|
||||
exit 1
|
||||
fi
|
||||
SEEN_DEBUG_TESTS[$requested_test]=1
|
||||
|
||||
case "$requested_test" in
|
||||
ready)
|
||||
DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
|
||||
;;
|
||||
merge)
|
||||
DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
|
||||
echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
|
||||
exit 1
|
||||
elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
|
||||
TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
|
||||
else
|
||||
TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
|
||||
python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
output_path, *suite_specs = sys.argv[1:]
|
||||
combined = {"env": {}, "steps": []}
|
||||
|
||||
for suite_spec in suite_specs:
|
||||
group_name, input_path = suite_spec.split(":", 1)
|
||||
with open(input_path, encoding="utf-8") as test_file:
|
||||
suite = yaml.safe_load(test_file)
|
||||
|
||||
for name, value in (suite.get("env") or {}).items():
|
||||
previous = combined["env"].get(name, value)
|
||||
if previous != value:
|
||||
raise ValueError(
|
||||
f"Conflicting environment value for {name}: {previous!r} != {value!r}"
|
||||
)
|
||||
combined["env"][name] = value
|
||||
|
||||
suite_steps = []
|
||||
for entry in suite.get("steps") or []:
|
||||
if "group" in entry:
|
||||
suite_steps.extend(entry.get("steps") or [])
|
||||
else:
|
||||
suite_steps.append(entry)
|
||||
combined["steps"].append({"group": group_name, "steps": suite_steps})
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as output_file:
|
||||
yaml.safe_dump(combined, output_file, sort_keys=False)
|
||||
PY
|
||||
fi
|
||||
echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
|
||||
elif [[ $BUILDKITE_BRANCH == "main" ]]; then
|
||||
TEST_YAML="test-amd-merge.yml"
|
||||
else
|
||||
TEST_YAML="test-amd-ready.yml"
|
||||
fi
|
||||
|
||||
cd .buildkite
|
||||
(
|
||||
set -x
|
||||
# Output pipeline.yaml with all blank lines removed
|
||||
minijinja-cli test-template.j2 "$TEST_YAML" \
|
||||
minijinja-cli test-template.j2 test-amd.yaml \
|
||||
-D branch="$BUILDKITE_BRANCH" \
|
||||
-D list_file_diff="$LIST_FILE_DIFF" \
|
||||
-D run_all="$RUN_ALL" \
|
||||
@@ -198,20 +108,19 @@ PY
|
||||
> pipeline.yaml
|
||||
)
|
||||
cat pipeline.yaml
|
||||
if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
|
||||
rm -f -- "$TEST_YAML"
|
||||
fi
|
||||
buildkite-agent artifact upload pipeline.yaml
|
||||
buildkite-agent pipeline upload pipeline.yaml
|
||||
exit 0
|
||||
}
|
||||
|
||||
get_diff() {
|
||||
git diff --name-only --diff-filter=ACMDR "$(git merge-base origin/main HEAD)"
|
||||
$(git add .)
|
||||
echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
|
||||
}
|
||||
|
||||
get_diff_main() {
|
||||
git diff --name-only --diff-filter=ACMDR HEAD~1
|
||||
$(git add .)
|
||||
echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
|
||||
}
|
||||
|
||||
file_diff=$(get_diff)
|
||||
@@ -219,13 +128,40 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
|
||||
file_diff=$(get_diff_main)
|
||||
fi
|
||||
|
||||
# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting.
|
||||
if [[ $BUILDKITE_BRANCH == "main" ]]; then
|
||||
gate_bootstrap_ci amd l3
|
||||
else
|
||||
gate_bootstrap_ci amd l2
|
||||
# ----------------------------------------------------------------------
|
||||
# Early exit start: skip pipeline if conditions are met
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
# skip pipeline if all changed files are under docs/
|
||||
if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
|
||||
if [[ -n "${file_diff:-}" ]]; then
|
||||
docs_only=1
|
||||
# Robust iteration over newline-separated file_diff
|
||||
while IFS= read -r f; do
|
||||
[[ -z "$f" ]] && continue
|
||||
# **Policy:** only skip if *every* path starts with docs/
|
||||
if [[ "$f" != docs/* ]]; then
|
||||
docs_only=0
|
||||
break
|
||||
fi
|
||||
done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
|
||||
|
||||
if [[ "$docs_only" -eq 1 ]]; then
|
||||
buildkite-agent annotate ":memo: CI skipped — docs/** only changes detected
|
||||
|
||||
\`\`\`
|
||||
${file_diff}
|
||||
\`\`\`" --style "info" || true
|
||||
echo "[docs-only] All changes are under docs/. Exiting before pipeline upload."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Early exit end
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
patterns=(
|
||||
"docker/Dockerfile"
|
||||
"CMakeLists.txt"
|
||||
@@ -250,7 +186,7 @@ for file in $file_diff; do
|
||||
# First check if file matches any pattern
|
||||
matches_pattern=0
|
||||
for pattern in "${patterns[@]}"; do
|
||||
if [[ $file == "$pattern"* ]] || [[ $file == "$pattern" ]]; then
|
||||
if [[ $file == $pattern* ]] || [[ $file == $pattern ]]; then
|
||||
matches_pattern=1
|
||||
break
|
||||
fi
|
||||
@@ -260,7 +196,7 @@ for file in $file_diff; do
|
||||
if [[ $matches_pattern -eq 1 ]]; then
|
||||
matches_ignore=0
|
||||
for ignore in "${ignore_patterns[@]}"; do
|
||||
if [[ $file == "$ignore"* ]] || [[ $file == "$ignore" ]]; then
|
||||
if [[ $file == $ignore* ]] || [[ $file == $ignore ]]; then
|
||||
matches_ignore=1
|
||||
break
|
||||
fi
|
||||
@@ -1,648 +0,0 @@
|
||||
# Uploader-only hardware presets for CUDA / NPU test YAML.
|
||||
# Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to
|
||||
# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins`` (+ ``retry``
|
||||
# for ``l4_*``), then strips ``mirror_hardwares`` before Buildkite upload.
|
||||
#
|
||||
# Omit the key: compose {chip}_{n} from pytest -m SKU + cards_n (max of several
|
||||
# positives; not cards_1 with no positive cards_* → that chip's highest existing preset).
|
||||
# Unset MIRROR_HW matches H100 or L4 in -m (both → H100); no match skips the step.
|
||||
# MIRROR_HW=b200 matches B200 in -m (else the step is skipped).
|
||||
# MIRROR_HW must be empty or b200 (case-insensitive); unknown values fail the upload.
|
||||
# CUDA preset strings (h100_4, l4_1) are omitted when MIRROR_HW=b200.
|
||||
# Unset MIRROR_HW keeps string presets. NPU presets ignore MIRROR_HW.
|
||||
#
|
||||
# CUDA naming: {chip}_{count}
|
||||
# l4_1, l4_2, l4_3, l4_4, h100_1 … h100_4, b200_1 … b200_4
|
||||
# NPU naming: {queue}_{resource_class}
|
||||
# a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
|
||||
# a2b3_npu_4 — ascend-a2b3, resource_class npu-4
|
||||
# a3_npu_2 — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
|
||||
#
|
||||
# B200 (b200-k8s): matches vLLM device ``b200-k8s`` from ci-infra
|
||||
# (AgentQueue.B200_K8S + b200_plugin_template). Queue routes the K8s pool;
|
||||
# podSpec uses hostNetwork + /raid|/mnt/shared (no H100-style nodeSelector).
|
||||
#
|
||||
# The ``l4_*`` presets run on the EKS ``l4-k8s`` queue (agent-stack-k8s),
|
||||
# packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU /
|
||||
# memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm).
|
||||
# Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137).
|
||||
|
||||
x-npu-k8s-plugins: &npu_k8s_plugins
|
||||
- kubernetes:
|
||||
podSpecPatch:
|
||||
imagePullSecrets:
|
||||
- name: swr-secret
|
||||
|
||||
# Infra-only retry. Omit exit 1 (pytest failed) and 137 (cgroup OOM / SIGKILL
|
||||
# of the test process). ci-infra ``K8S_RETRY`` still retries exit 1.
|
||||
x-l4-k8s-retry: &l4_k8s_retry
|
||||
automatic:
|
||||
- exit_status: -1
|
||||
limit: 1
|
||||
- exit_status: 128
|
||||
limit: 1
|
||||
- signal_reason: agent_stop
|
||||
limit: 1
|
||||
- signal_reason: agent_refused
|
||||
limit: 1
|
||||
|
||||
|
||||
mirror_hardwares:
|
||||
l4_1:
|
||||
agents:
|
||||
queue: l4-k8s
|
||||
retry: *l4_k8s_retry
|
||||
plugins:
|
||||
- kubernetes:
|
||||
metadata:
|
||||
annotations:
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
|
||||
karpenter.sh/do-not-disrupt: "true"
|
||||
podSpec:
|
||||
priorityClassName: ci
|
||||
serviceAccountName: buildkite-gpu-jobs
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
nodeSelector:
|
||||
vllm.ci/gpu-pool: l4x4
|
||||
containers:
|
||||
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
requests:
|
||||
nvidia.com/gpu: 1
|
||||
cpu: "10"
|
||||
memory: 40Gi
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /fsx/hf_cache
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /fsx/hf_cache
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 24Gi
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /fsx/hf_cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
l4_2:
|
||||
agents:
|
||||
queue: l4-k8s
|
||||
retry: *l4_k8s_retry
|
||||
plugins:
|
||||
- kubernetes:
|
||||
metadata:
|
||||
annotations:
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
|
||||
karpenter.sh/do-not-disrupt: "true"
|
||||
podSpec:
|
||||
priorityClassName: ci
|
||||
serviceAccountName: buildkite-gpu-jobs
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
nodeSelector:
|
||||
vllm.ci/gpu-pool: l4x4
|
||||
containers:
|
||||
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
requests:
|
||||
nvidia.com/gpu: 2
|
||||
cpu: "20"
|
||||
memory: 80Gi
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /fsx/hf_cache
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /fsx/hf_cache
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 48Gi
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /fsx/hf_cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
l4_3:
|
||||
agents:
|
||||
queue: l4-k8s
|
||||
retry: *l4_k8s_retry
|
||||
plugins:
|
||||
- kubernetes:
|
||||
metadata:
|
||||
annotations:
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
|
||||
karpenter.sh/do-not-disrupt: "true"
|
||||
podSpec:
|
||||
priorityClassName: ci
|
||||
serviceAccountName: buildkite-gpu-jobs
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
nodeSelector:
|
||||
vllm.ci/gpu-pool: l4x4
|
||||
containers:
|
||||
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 3
|
||||
requests:
|
||||
nvidia.com/gpu: 3
|
||||
cpu: "30"
|
||||
memory: 120Gi
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /fsx/hf_cache
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /fsx/hf_cache
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 72Gi
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /fsx/hf_cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
l4_4:
|
||||
agents:
|
||||
queue: l4-k8s
|
||||
retry: *l4_k8s_retry
|
||||
plugins:
|
||||
- kubernetes:
|
||||
metadata:
|
||||
annotations:
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
|
||||
karpenter.sh/do-not-disrupt: "true"
|
||||
podSpec:
|
||||
priorityClassName: ci
|
||||
serviceAccountName: buildkite-gpu-jobs
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
nodeSelector:
|
||||
vllm.ci/gpu-pool: l4x4
|
||||
containers:
|
||||
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 4
|
||||
requests:
|
||||
nvidia.com/gpu: 4
|
||||
cpu: "40"
|
||||
memory: 160Gi
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /fsx/hf_cache
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /fsx/hf_cache
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 96Gi
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /fsx/hf_cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
h100_1:
|
||||
agents:
|
||||
queue: mithril-h100-pool
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
h100_2:
|
||||
agents:
|
||||
queue: mithril-h100-pool
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
h100_3:
|
||||
agents:
|
||||
queue: mithril-h100-pool
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 3
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
h100_4:
|
||||
agents:
|
||||
queue: mithril-h100-pool
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 4
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
# vLLM device: b200-k8s → agents.queue b200-k8s + Buildkite K8s agent-stack podSpec
|
||||
b200_1:
|
||||
agents:
|
||||
queue: b200-k8s
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
runtimeClassName: nvidia
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
imagePullSecrets:
|
||||
- name: k8s-ecr-login-renew-docker-secret
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- IPC_LOCK
|
||||
- SYS_RESOURCE
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: raid
|
||||
mountPath: /raid
|
||||
- name: shared
|
||||
mountPath: /mnt/shared
|
||||
env:
|
||||
- name: NCCL_CUMEM_HOST_ENABLE
|
||||
value: "0"
|
||||
- name: HF_HOME
|
||||
value: /raid/hf_cache
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: raid
|
||||
hostPath:
|
||||
path: /raid
|
||||
type: DirectoryOrCreate
|
||||
- name: shared
|
||||
hostPath:
|
||||
path: /mnt/shared
|
||||
type: DirectoryOrCreate
|
||||
|
||||
b200_2:
|
||||
agents:
|
||||
queue: b200-k8s
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
runtimeClassName: nvidia
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
imagePullSecrets:
|
||||
- name: k8s-ecr-login-renew-docker-secret
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- IPC_LOCK
|
||||
- SYS_RESOURCE
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: raid
|
||||
mountPath: /raid
|
||||
- name: shared
|
||||
mountPath: /mnt/shared
|
||||
env:
|
||||
- name: NCCL_CUMEM_HOST_ENABLE
|
||||
value: "0"
|
||||
- name: HF_HOME
|
||||
value: /raid/hf_cache
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: raid
|
||||
hostPath:
|
||||
path: /raid
|
||||
type: DirectoryOrCreate
|
||||
- name: shared
|
||||
hostPath:
|
||||
path: /mnt/shared
|
||||
type: DirectoryOrCreate
|
||||
|
||||
b200_3:
|
||||
agents:
|
||||
queue: b200-k8s
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
runtimeClassName: nvidia
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
imagePullSecrets:
|
||||
- name: k8s-ecr-login-renew-docker-secret
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 3
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- IPC_LOCK
|
||||
- SYS_RESOURCE
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: raid
|
||||
mountPath: /raid
|
||||
- name: shared
|
||||
mountPath: /mnt/shared
|
||||
env:
|
||||
- name: NCCL_CUMEM_HOST_ENABLE
|
||||
value: "0"
|
||||
- name: HF_HOME
|
||||
value: /raid/hf_cache
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: raid
|
||||
hostPath:
|
||||
path: /raid
|
||||
type: DirectoryOrCreate
|
||||
- name: shared
|
||||
hostPath:
|
||||
path: /mnt/shared
|
||||
type: DirectoryOrCreate
|
||||
|
||||
b200_4:
|
||||
agents:
|
||||
queue: b200-k8s
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
runtimeClassName: nvidia
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
imagePullSecrets:
|
||||
- name: k8s-ecr-login-renew-docker-secret
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 4
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- IPC_LOCK
|
||||
- SYS_RESOURCE
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: raid
|
||||
mountPath: /raid
|
||||
- name: shared
|
||||
mountPath: /mnt/shared
|
||||
env:
|
||||
- name: NCCL_CUMEM_HOST_ENABLE
|
||||
value: "0"
|
||||
- name: HF_HOME
|
||||
value: /raid/hf_cache
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: raid
|
||||
hostPath:
|
||||
path: /raid
|
||||
type: DirectoryOrCreate
|
||||
- name: shared
|
||||
hostPath:
|
||||
path: /mnt/shared
|
||||
type: DirectoryOrCreate
|
||||
|
||||
a2b3_npu_1:
|
||||
agents:
|
||||
queue: "ascend-a2b3"
|
||||
resource_class: "npu-1"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
|
||||
plugins: *npu_k8s_plugins
|
||||
|
||||
a2b3_npu_4:
|
||||
agents:
|
||||
queue: "ascend-a2b3"
|
||||
resource_class: "npu-4"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
|
||||
plugins: *npu_k8s_plugins
|
||||
|
||||
|
||||
a2b3_npu_8:
|
||||
agents:
|
||||
queue: "ascend-a2b3"
|
||||
resource_class: "npu-8"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
|
||||
plugins: *npu_k8s_plugins
|
||||
|
||||
a3_npu_2:
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-2"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins: *npu_k8s_plugins
|
||||
|
||||
a3_npu_4:
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-4"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins: *npu_k8s_plugins
|
||||
|
||||
a3_npu_8:
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-8"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins: *npu_k8s_plugins
|
||||
|
||||
a3_npu_16:
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-16"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins: *npu_k8s_plugins
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared Buildkite skip-ci helpers for AMD / Intel bootstrap scripts.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SKIP_CI_PY="${SCRIPT_DIR}/skip_ci.py"
|
||||
|
||||
# Resolve git diff once. Exit 0 when this bootstrap should stop uploading.
|
||||
# Decision details are logged by skip_ci.py; this helper only annotates skip-all.
|
||||
# Usage: gate_bootstrap_ci <platform> <l2|l3>
|
||||
gate_bootstrap_ci() {
|
||||
local platform="$1"
|
||||
local level="$2"
|
||||
local kind
|
||||
if kind="$(python3 "${SKIP_CI_PY}" gate "${platform}" "${level}")"; then
|
||||
if [[ "${kind}" == "skip-all" ]]; then
|
||||
buildkite-agent annotate \
|
||||
":memo: CI skipped — docs or pytest skip-mark changes only" \
|
||||
--style "info" 2>/dev/null || true
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run a model's E2E tests once per entry mode as separate coverage runs and upload
|
||||
# the reports, so per-mode attribution survives (see vllm-project/vllm-omni#5332).
|
||||
#
|
||||
# Usage: run_cov_split.sh --model-id <id> [--offline <paths>] [--online <paths>] \
|
||||
# -- <pytest args...>
|
||||
#
|
||||
# --model-id Names the artifacts: coverage-<id>-{offline,online}-<step id>.xml.gz.
|
||||
# Use the model's directory name under vllm_omni/*/models/.
|
||||
# --offline Pytest path(s) for the offline half. Repeatable.
|
||||
# --online Pytest path(s) for the online half. Repeatable.
|
||||
# -- Everything after this is forwarded verbatim to every pytest run,
|
||||
# quoting preserved: the job's own -m expression, --run-level, and
|
||||
# anything else it needs (e.g. --test-config-file).
|
||||
#
|
||||
# At least one of --offline/--online is required, so a job with only one mode runs
|
||||
# just that half. Total runtime is bounded by the step's timeout_in_minutes.
|
||||
#
|
||||
# Coverage + artifact upload run only on scheduled weekly E2E sweeps
|
||||
# (WEEKLY=1 and BUILDKITE_BRANCH=main — same gate as bootstrap --e2e).
|
||||
# All other invocations (PR merge-test, regular main merge, local) run the
|
||||
# same combined pytest as the job used before coverage was added.
|
||||
#
|
||||
# Exits non-zero if any half or the upload failed.
|
||||
set -uo pipefail
|
||||
|
||||
MODEL_ID=""
|
||||
OFFLINE=()
|
||||
ONLINE=()
|
||||
PYTEST_ARGS=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--model-id) MODEL_ID="$2"; shift 2 ;;
|
||||
--offline) OFFLINE+=("$2"); shift 2 ;;
|
||||
--online) ONLINE+=("$2"); shift 2 ;;
|
||||
--) shift; PYTEST_ARGS=("$@"); break ;;
|
||||
*) echo "run_cov_split.sh: unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "${MODEL_ID}" ]]; then
|
||||
echo "run_cov_split.sh: missing --model-id" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ((${#OFFLINE[@]} == 0 && ${#ONLINE[@]} == 0)); then
|
||||
echo "run_cov_split.sh: need at least one of --offline / --online" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
_should_run_coverage() {
|
||||
[[ "${WEEKLY:-}" == "1" && "${BUILDKITE_BRANCH:-}" == "main" ]]
|
||||
}
|
||||
|
||||
if ! _should_run_coverage; then
|
||||
PATHS=("${OFFLINE[@]}" "${ONLINE[@]}")
|
||||
echo "--- pytest (no coverage): ${MODEL_ID}"
|
||||
pytest -s -v "${PATHS[@]}" "${PYTEST_ARGS[@]}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# Two steps can test the same model, and artifacts are scoped to the build (a
|
||||
# nightly one carries the ready, merge and nightly tiers at once), so unsuffixed
|
||||
# reports would share a path -- a glob download then races to rename onto one file.
|
||||
# Not BUILDKITE_JOB_ID: the step id survives retries, so attempts collapse under
|
||||
# Buildkite's latest-attempt lookup instead of accumulating. A parallel step would
|
||||
# need BUILDKITE_PARALLEL_JOB as well; no pilot is one.
|
||||
REPORT_SUFFIX="${BUILDKITE_STEP_ID:-local}"
|
||||
OFFLINE_REPORT="coverage-${MODEL_ID}-offline-${REPORT_SUFFIX}.xml"
|
||||
ONLINE_REPORT="coverage-${MODEL_ID}-online-${REPORT_SUFFIX}.xml"
|
||||
|
||||
run_half() {
|
||||
local mode="$1"
|
||||
shift
|
||||
local report="coverage-${MODEL_ID}-${mode}-${REPORT_SUFFIX}.xml"
|
||||
echo "--- coverage: ${MODEL_ID} ${mode}"
|
||||
pytest -s -v "$@" "${PYTEST_ARGS[@]}" --cov=vllm_omni --cov-report="xml:${report}"
|
||||
local status=$?
|
||||
# Cobertura lists every statement in vllm_omni whether or not it ran, so the
|
||||
# report is ~7MB no matter how little the job covers. The repetition is what
|
||||
# dominates it, and gzip takes that to ~400KB.
|
||||
gzip -9 -f "${report}" || status=1
|
||||
return "${status}"
|
||||
}
|
||||
|
||||
# Clear both modes, not just the ones being run: the upload glob matches both, so
|
||||
# a report left by an earlier run would otherwise be published as if the mode it
|
||||
# belongs to had run this time.
|
||||
rm -f "${OFFLINE_REPORT}" "${ONLINE_REPORT}" "${OFFLINE_REPORT}.gz" "${ONLINE_REPORT}.gz"
|
||||
|
||||
EXIT=0
|
||||
((${#OFFLINE[@]})) && { run_half offline "${OFFLINE[@]}" || EXIT=1; }
|
||||
((${#ONLINE[@]})) && { run_half online "${ONLINE[@]}" || EXIT=1; }
|
||||
|
||||
buildkite-agent artifact upload "coverage-${MODEL_ID}-*-${REPORT_SUFFIX}.xml.gz" || EXIT=1
|
||||
|
||||
exit "${EXIT}"
|
||||
@@ -1,584 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared Buildkite skip-ci and CI-YAML level targeting logic.
|
||||
|
||||
``resolve_ci_decision(changed_files, ...)`` buckets explicit file lists (tests, dry-run).
|
||||
``resolve_ci_context_from_git()`` resolves git diff once and returns decision + changed files
|
||||
for Buildkite bootstrap / upload scripts.
|
||||
|
||||
CLI (for bootstrap scripts; exit code is the signal):
|
||||
python3 skip_ci.py gate <platform> <level>
|
||||
Exit 0 when bootstrap should stop (skip-all, or that L2/L3 target is off).
|
||||
Prints ``skip-all`` or ``skip-l23`` on stdout; logs the decision once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
|
||||
LOG = "[skip-ci]"
|
||||
ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
||||
|
||||
PLATFORMS = ("cuda", "amd", "intel", "npu")
|
||||
L23_LEVELS = frozenset({"l2", "l3"})
|
||||
|
||||
L2_YAML_FILES: dict[str, str] = {
|
||||
".buildkite/cuda/test-ready.yml": "cuda",
|
||||
".buildkite/npu/test-npu-ready.yml": "npu",
|
||||
".buildkite/amd/test-amd-ready.yml": "amd",
|
||||
".buildkite/intel/pipeline-intel.yml": "intel",
|
||||
}
|
||||
|
||||
L3_YAML_FILES: dict[str, str] = {
|
||||
".buildkite/cuda/test-merge.yml": "cuda",
|
||||
".buildkite/amd/test-amd-merge.yml": "amd",
|
||||
}
|
||||
|
||||
L45_YAML_FILES: dict[str, str] = {
|
||||
".buildkite/cuda/test-nightly.yml": "cuda",
|
||||
".buildkite/cuda/test-weekly.yml": "cuda",
|
||||
".buildkite/npu/test-npu-nightly.yml": "npu",
|
||||
}
|
||||
|
||||
|
||||
def _empty_platform_buckets() -> dict[str, list[str]]:
|
||||
return {platform: [] for platform in PLATFORMS}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CiDevice:
|
||||
"""Per-platform / per-level skip flags for yaml-gated bootstrap (``skip_l2_l3``)."""
|
||||
|
||||
skip_cuda_l2: bool = False
|
||||
skip_cuda_l3: bool = False
|
||||
skip_amd_l2: bool = False
|
||||
skip_amd_l3: bool = False
|
||||
skip_intel_l2: bool = False
|
||||
skip_intel_l3: bool = False
|
||||
skip_npu_l2: bool = False
|
||||
skip_npu_l3: bool = False
|
||||
|
||||
def is_skip(self, platform: str, level: str) -> bool:
|
||||
return bool(getattr(self, f"skip_{platform}_{level}", True))
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiffBuckets:
|
||||
"""Changed files grouped by doc / skip-mark / CI yaml / other."""
|
||||
|
||||
docs: list[str] = field(default_factory=list)
|
||||
skip_tests: list[str] = field(default_factory=list)
|
||||
other: list[str] = field(default_factory=list)
|
||||
l2: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
|
||||
l3: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
|
||||
l45: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
|
||||
|
||||
@property
|
||||
def has_doc_changes(self) -> bool:
|
||||
return bool(self.docs)
|
||||
|
||||
@property
|
||||
def has_skip_test_changes(self) -> bool:
|
||||
return bool(self.skip_tests)
|
||||
|
||||
@property
|
||||
def has_other_changes(self) -> bool:
|
||||
"""Product code or paths outside skip-ci scope."""
|
||||
return bool(self.other)
|
||||
|
||||
@property
|
||||
def has_l23_yaml_changes(self) -> bool:
|
||||
"""At least one whitelisted L2/L3 CI yaml file changed."""
|
||||
return any(self.l2[platform] or self.l3[platform] for platform in PLATFORMS)
|
||||
|
||||
@property
|
||||
def has_l45_yaml_changes(self) -> bool:
|
||||
"""At least one whitelisted L4/L5 (nightly/weekly) CI yaml file changed."""
|
||||
return any(self.l45[platform] for platform in PLATFORMS)
|
||||
|
||||
@property
|
||||
def has_skip_ci_scope_changes(self) -> bool:
|
||||
"""At least one doc / skip-mark / whitelisted CI yaml change."""
|
||||
return (
|
||||
self.has_doc_changes or self.has_skip_test_changes or self.has_l23_yaml_changes or self.has_l45_yaml_changes
|
||||
)
|
||||
|
||||
def l2_platforms(self) -> frozenset[str]:
|
||||
return frozenset(platform for platform in PLATFORMS if self.l2[platform])
|
||||
|
||||
def l3_platforms(self) -> frozenset[str]:
|
||||
return frozenset(platform for platform in PLATFORMS if self.l3[platform])
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkipL2L3Basis:
|
||||
skip_all_l2: bool
|
||||
skip_all_l3: bool
|
||||
|
||||
@classmethod
|
||||
def from_buckets(cls, buckets: DiffBuckets) -> SkipL2L3Basis | None:
|
||||
"""L4/L5 yaml → skip both L2 and L3 (L2/L3 buckets may rescue).
|
||||
|
||||
Docs / skip-mark changes do not block this path; they are ignored for
|
||||
L2/L3 targeting when CI YAML is also present.
|
||||
"""
|
||||
if buckets.has_other_changes:
|
||||
return None
|
||||
if not buckets.has_l45_yaml_changes:
|
||||
return None
|
||||
return cls(
|
||||
skip_all_l2=not buckets.l2_platforms(),
|
||||
skip_all_l3=not buckets.l3_platforms(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkipL3Basis:
|
||||
enable_l2_platforms: frozenset[str]
|
||||
|
||||
@classmethod
|
||||
def from_buckets(cls, buckets: DiffBuckets) -> SkipL3Basis | None:
|
||||
"""L2 yaml changed → skip L3; keep L2 on touched platforms."""
|
||||
platforms = buckets.l2_platforms()
|
||||
if not platforms:
|
||||
return None
|
||||
return cls(enable_l2_platforms=platforms)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkipL2Basis:
|
||||
enable_l3_platforms: frozenset[str]
|
||||
|
||||
@classmethod
|
||||
def from_buckets(cls, buckets: DiffBuckets) -> SkipL2Basis | None:
|
||||
"""L3 yaml changed → skip L2; keep L3 on touched platforms."""
|
||||
platforms = buckets.l3_platforms()
|
||||
if not platforms:
|
||||
return None
|
||||
return cls(enable_l3_platforms=platforms)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CiDecision:
|
||||
"""Final CI skip decision for one diff."""
|
||||
|
||||
skip_all: bool = False
|
||||
skip_l2_l3: bool = False
|
||||
device: CiDevice = field(default_factory=CiDevice)
|
||||
message: str = ""
|
||||
|
||||
@classmethod
|
||||
def skip_all_basis(cls, buckets: DiffBuckets, *, diff_range: str | None) -> bool:
|
||||
"""Docs / pytest skip-mark only → suppress entire default CI."""
|
||||
if buckets.has_other_changes:
|
||||
return False
|
||||
if (buckets.has_l23_yaml_changes or buckets.has_l45_yaml_changes) and (
|
||||
buckets.has_doc_changes or buckets.has_skip_test_changes
|
||||
):
|
||||
return False
|
||||
if not buckets.has_doc_changes and not buckets.has_skip_test_changes:
|
||||
return False
|
||||
if diff_range is None and not buckets.has_doc_changes:
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def from_yaml_basis(
|
||||
cls,
|
||||
l2_l3_basis: SkipL2L3Basis | None,
|
||||
l3_basis: SkipL3Basis | None,
|
||||
l2_basis: SkipL2Basis | None,
|
||||
) -> CiDecision:
|
||||
"""Build a yaml-gated decision by merging L2/L3 skip bases.
|
||||
|
||||
Bases are applied as a union of enables: L2 yaml keeps L2 on touched
|
||||
platforms, L3 yaml keeps L3 on touched platforms. Cross-platform mixes
|
||||
(e.g. CUDA ready + AMD merge) must preserve both enables. The resulting
|
||||
per-platform / per-level matrix is stored on ``device`` as ``skip_*``.
|
||||
"""
|
||||
run = {platform: {"l2": True, "l3": True} for platform in PLATFORMS}
|
||||
|
||||
if l2_l3_basis is not None:
|
||||
if l2_l3_basis.skip_all_l2:
|
||||
for platform in PLATFORMS:
|
||||
run[platform]["l2"] = False
|
||||
if l2_l3_basis.skip_all_l3:
|
||||
for platform in PLATFORMS:
|
||||
run[platform]["l3"] = False
|
||||
|
||||
if l3_basis is not None:
|
||||
for platform in PLATFORMS:
|
||||
run[platform]["l3"] = False
|
||||
run[platform]["l2"] = platform in l3_basis.enable_l2_platforms
|
||||
|
||||
if l2_basis is not None:
|
||||
for platform in PLATFORMS:
|
||||
if platform in l2_basis.enable_l3_platforms:
|
||||
run[platform]["l3"] = True
|
||||
if l3_basis is None or platform not in l3_basis.enable_l2_platforms:
|
||||
run[platform]["l2"] = False
|
||||
elif l3_basis is None:
|
||||
# L3-yaml-only: disable platforms that did not change L3.
|
||||
run[platform]["l2"] = False
|
||||
run[platform]["l3"] = False
|
||||
# else: leave L2 enables from SkipL3Basis; keep this platform's L3 off.
|
||||
|
||||
return cls(
|
||||
skip_l2_l3=True,
|
||||
device=cls._device_from_run(run),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _device_from_run(run: dict[str, dict[str, bool]]) -> CiDevice:
|
||||
return CiDevice(
|
||||
**{f"skip_{platform}_{level}": not run[platform][level] for platform in PLATFORMS for level in ("l2", "l3")}
|
||||
)
|
||||
|
||||
def is_run(self, platform: str, level: str) -> bool:
|
||||
if self.skip_all:
|
||||
return False
|
||||
if not self.skip_l2_l3:
|
||||
return True
|
||||
if level not in L23_LEVELS:
|
||||
return True
|
||||
return not self.device.is_skip(platform, level)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CiContext:
|
||||
"""Skip-ci decision plus changed files used for pipeline filtering."""
|
||||
|
||||
decision: CiDecision
|
||||
changed_files: list[str] | None
|
||||
|
||||
|
||||
def _log(message: str) -> None:
|
||||
print(f"{LOG}: {message}", file=sys.stderr)
|
||||
|
||||
|
||||
def _git(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
# --- Diff helpers ---
|
||||
|
||||
|
||||
def _resolve_diff_range() -> str | None:
|
||||
"""Return a git diff range for PR or main builds, or None when unavailable."""
|
||||
is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
|
||||
"BUILDKITE_PULL_REQUEST", ""
|
||||
)
|
||||
commit = os.environ.get("BUILDKITE_COMMIT", "")
|
||||
|
||||
if is_pr:
|
||||
base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main")
|
||||
base_ref = f"origin/{base_branch}"
|
||||
if _git("rev-parse", "--verify", base_ref).returncode != 0:
|
||||
_log(f"origin/{base_branch} not found locally; trying fetch")
|
||||
_git("fetch", "--depth=200", "origin", base_branch)
|
||||
if _git("rev-parse", "--verify", base_ref).returncode != 0:
|
||||
if _git("rev-parse", "--verify", base_branch).returncode == 0:
|
||||
base_ref = base_branch
|
||||
else:
|
||||
_log(f"cannot resolve PR base {base_branch}; using safe defaults")
|
||||
return None
|
||||
return f"{base_ref}...{commit}"
|
||||
|
||||
if os.environ.get("BUILDKITE_BRANCH", "") == "main":
|
||||
if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
|
||||
_log("main commit has no parent; using safe defaults")
|
||||
return None
|
||||
return f"{commit}^..{commit}"
|
||||
|
||||
_log("not PR/main build; using safe defaults")
|
||||
return None
|
||||
|
||||
|
||||
def _changed_files_for_diff_range(diff_range: str | None) -> list[str] | None:
|
||||
"""Return changed file paths for *diff_range*, or None when unavailable."""
|
||||
if diff_range is None:
|
||||
return None
|
||||
|
||||
result = _git("diff", "--name-only", diff_range)
|
||||
if result.returncode != 0:
|
||||
_log(f"git diff failed ({diff_range}); using safe defaults")
|
||||
return None
|
||||
|
||||
files = [line for line in result.stdout.splitlines() if line.strip()]
|
||||
_log(f"{len(files)} changed file(s)")
|
||||
return files
|
||||
|
||||
|
||||
def resolve_ci_context_from_git() -> CiContext:
|
||||
"""Resolve git diff once and return skip-ci decision plus changed files."""
|
||||
diff_range = _resolve_diff_range()
|
||||
changed_files = _changed_files_for_diff_range(diff_range)
|
||||
decision = resolve_ci_decision(changed_files, diff_range=diff_range)
|
||||
return CiContext(decision=decision, changed_files=changed_files)
|
||||
|
||||
|
||||
# --- Doc / skip-mark helpers ---
|
||||
|
||||
|
||||
_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(")
|
||||
_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b")
|
||||
_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+")
|
||||
|
||||
|
||||
def _diff_only_contains_skip_mark_changes(diff_text: str) -> bool:
|
||||
"""Return True when a unified diff only *adds* pytest skip marks.
|
||||
|
||||
Deleting a skip mark re-enables the test and must not suppress CI. Editing an
|
||||
existing skip (``-`` old / ``+`` new) still qualifies when at least one skip
|
||||
mark is added.
|
||||
"""
|
||||
|
||||
def paren_balance(line: str) -> int:
|
||||
return line.count("(") - line.count(")")
|
||||
|
||||
def is_blank_or_comment(line: str) -> bool:
|
||||
stripped = line.strip()
|
||||
return not stripped or stripped.startswith("#")
|
||||
|
||||
def is_skip_mark_line(line: str) -> bool:
|
||||
return _SKIP_MARK_RE.search(line.strip()) is not None
|
||||
|
||||
def is_pytestmark_adjacent_line(line: str) -> bool:
|
||||
stripped = line.strip().rstrip(",")
|
||||
if not stripped or stripped in {"[", "]"} or stripped.startswith("#"):
|
||||
return True
|
||||
if stripped.startswith("pytestmark"):
|
||||
return True
|
||||
return _PYTEST_MARK_ONLY_RE.search(stripped) is not None
|
||||
|
||||
pending_depth = 0
|
||||
saw_change = False
|
||||
has_added_skip_mark = False
|
||||
for raw_line in diff_text.splitlines():
|
||||
if raw_line.startswith("@@"):
|
||||
pending_depth = 0
|
||||
continue
|
||||
if not (raw_line.startswith("+") or raw_line.startswith("-")):
|
||||
continue
|
||||
if raw_line.startswith("+++") or raw_line.startswith("---"):
|
||||
continue
|
||||
|
||||
saw_change = True
|
||||
is_add = raw_line.startswith("+")
|
||||
content = raw_line[1:]
|
||||
|
||||
if pending_depth > 0:
|
||||
pending_depth += paren_balance(content)
|
||||
if pending_depth < 0:
|
||||
pending_depth = 0
|
||||
continue
|
||||
|
||||
if is_blank_or_comment(content):
|
||||
continue
|
||||
|
||||
if is_skip_mark_line(content):
|
||||
if is_add:
|
||||
has_added_skip_mark = True
|
||||
pending_depth = max(0, paren_balance(content))
|
||||
continue
|
||||
|
||||
if is_pytestmark_adjacent_line(content):
|
||||
continue
|
||||
|
||||
return False
|
||||
|
||||
return saw_change and has_added_skip_mark
|
||||
|
||||
|
||||
def _is_skip_testcase_change(file_path: str, *, diff_range: str) -> bool:
|
||||
"""Return True when a test file diff is skip-mark-only (including new-file cases)."""
|
||||
diff_result = _git("diff", diff_range, "--", file_path)
|
||||
if diff_result.returncode != 0:
|
||||
_log(f"skip-mark-only: git diff failed for {file_path}")
|
||||
return False
|
||||
|
||||
diff_text = diff_result.stdout
|
||||
if not diff_text.strip():
|
||||
_log(f"skip-mark-only: empty diff for {file_path}")
|
||||
return False
|
||||
|
||||
if _diff_only_contains_skip_mark_changes(diff_text):
|
||||
return True
|
||||
|
||||
if "new file mode" not in diff_text and "--- /dev/null" not in diff_text:
|
||||
_log(f"skip-mark-only: non-skip changes in {file_path}")
|
||||
return False
|
||||
|
||||
commit = os.environ.get("BUILDKITE_COMMIT", "")
|
||||
if not commit:
|
||||
return False
|
||||
|
||||
content_result = _git("show", f"{commit}:{file_path}")
|
||||
if content_result.returncode != 0:
|
||||
_log(f"skip-mark-only: cannot read {file_path} at {commit}")
|
||||
return False
|
||||
|
||||
content = content_result.stdout
|
||||
match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL)
|
||||
if match is not None and _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None:
|
||||
_log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
|
||||
return True
|
||||
if re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE):
|
||||
_log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
|
||||
return True
|
||||
|
||||
_log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}")
|
||||
return False
|
||||
|
||||
|
||||
def _classify_changed_files_into_buckets(changed_files: list[str], *, diff_range: str | None) -> DiffBuckets:
|
||||
"""Classify each changed file path into a ``DiffBuckets`` category."""
|
||||
buckets = DiffBuckets()
|
||||
for file_path in changed_files:
|
||||
if not file_path:
|
||||
continue
|
||||
if file_path.startswith("docs/") or file_path.endswith(".md") or file_path == "mkdocs.yml":
|
||||
buckets.docs.append(file_path)
|
||||
continue
|
||||
path = Path(file_path)
|
||||
if path.suffix == ".py" and path.parts and path.parts[0] == "tests":
|
||||
if diff_range is not None and _is_skip_testcase_change(file_path, diff_range=diff_range):
|
||||
buckets.skip_tests.append(file_path)
|
||||
else:
|
||||
buckets.other.append(file_path)
|
||||
continue
|
||||
if file_path in L2_YAML_FILES:
|
||||
platform = L2_YAML_FILES[file_path]
|
||||
buckets.l2[platform].append(file_path)
|
||||
elif file_path in L3_YAML_FILES:
|
||||
platform = L3_YAML_FILES[file_path]
|
||||
buckets.l3[platform].append(file_path)
|
||||
elif file_path in L45_YAML_FILES:
|
||||
platform = L45_YAML_FILES[file_path]
|
||||
buckets.l45[platform].append(file_path)
|
||||
else:
|
||||
buckets.other.append(file_path)
|
||||
return buckets
|
||||
|
||||
|
||||
# --- Decision ---
|
||||
|
||||
|
||||
def _finish(decision: CiDecision, message: str) -> CiDecision:
|
||||
finished = replace(decision, message=message)
|
||||
_log(message)
|
||||
return finished
|
||||
|
||||
|
||||
def _format_yaml_gated_message(buckets: DiffBuckets, decision: CiDecision) -> str:
|
||||
"""Human-readable summary of which CI YAML changed and which L2/L3 targets run."""
|
||||
changed_parts: list[str] = []
|
||||
for level_name, by_platform in (("L2", buckets.l2), ("L3", buckets.l3), ("L4/L5", buckets.l45)):
|
||||
files = [path for platform in PLATFORMS for path in by_platform[platform]]
|
||||
if files:
|
||||
changed_parts.append(f"{level_name}=[{', '.join(files)}]")
|
||||
|
||||
run_targets = [
|
||||
f"{platform}/{level}" for platform in PLATFORMS for level in ("l2", "l3") if decision.is_run(platform, level)
|
||||
]
|
||||
skip_targets = [
|
||||
f"{platform}/{level}"
|
||||
for platform in PLATFORMS
|
||||
for level in ("l2", "l3")
|
||||
if not decision.is_run(platform, level)
|
||||
]
|
||||
|
||||
changed = "; ".join(changed_parts) if changed_parts else "none"
|
||||
run = ", ".join(run_targets) if run_targets else "none"
|
||||
skip = ", ".join(skip_targets) if skip_targets else "none"
|
||||
if buckets.has_doc_changes or buckets.has_skip_test_changes:
|
||||
return (
|
||||
"CI config YAML + docs/skip-mark changed — follow CI YAML gating; "
|
||||
f"changed: {changed}; run: {run}; skip: {skip}"
|
||||
)
|
||||
return f"only CI config YAML changed — changed: {changed}; run: {run}; skip: {skip}"
|
||||
|
||||
|
||||
def resolve_ci_decision(
|
||||
changed_files: list[str] | None = None,
|
||||
*,
|
||||
diff_range: str | None = None,
|
||||
) -> CiDecision:
|
||||
if changed_files is None:
|
||||
return _finish(CiDecision(), "could not resolve changed files; run normal CI")
|
||||
|
||||
buckets = _classify_changed_files_into_buckets(changed_files, diff_range=diff_range)
|
||||
if buckets.has_other_changes or not buckets.has_skip_ci_scope_changes:
|
||||
return _finish(CiDecision(), "product or non-whitelisted changes present; run normal CI")
|
||||
|
||||
if CiDecision.skip_all_basis(buckets, diff_range=diff_range):
|
||||
return _finish(
|
||||
CiDecision(skip_all=True),
|
||||
"CI skipped — docs or pytest skip-mark changes only",
|
||||
)
|
||||
|
||||
# Docs / skip-mark mixed with whitelisted CI YAML → same yaml-gated path
|
||||
# as CI-YAML-only (docs/skip-mark do not widen to normal CI).
|
||||
if not buckets.has_l23_yaml_changes and not buckets.has_l45_yaml_changes:
|
||||
if buckets.has_skip_test_changes and diff_range is None:
|
||||
message = (
|
||||
"only pytest skip-mark test changes detected, but no git diff range "
|
||||
"is available to verify them (skip-all requires diff_range); run normal CI"
|
||||
)
|
||||
elif buckets.has_skip_test_changes:
|
||||
message = "pytest skip-mark test changes did not qualify for skip-all and no ci yaml changed; run normal CI"
|
||||
else:
|
||||
message = "doc/skip-mark changes did not qualify for skip-all; run normal CI"
|
||||
return _finish(CiDecision(), message)
|
||||
|
||||
decision = CiDecision.from_yaml_basis(
|
||||
SkipL2L3Basis.from_buckets(buckets),
|
||||
SkipL3Basis.from_buckets(buckets),
|
||||
SkipL2Basis.from_buckets(buckets),
|
||||
)
|
||||
return _finish(decision, _format_yaml_gated_message(buckets, decision))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
gate_parser = subparsers.add_parser(
|
||||
"gate",
|
||||
help="exit 0 when bootstrap should stop (skip-all or PLATFORM/LEVEL disabled); logs once",
|
||||
)
|
||||
gate_parser.add_argument("platform", help="cuda, amd, intel, or npu")
|
||||
gate_parser.add_argument("level", choices=("l2", "l3"), help="test level")
|
||||
|
||||
args = parser.parse_args()
|
||||
decision = resolve_ci_context_from_git().decision
|
||||
|
||||
if args.command == "gate":
|
||||
if decision.skip_all:
|
||||
print("skip-all")
|
||||
return 0
|
||||
if not decision.is_run(args.platform, args.level):
|
||||
print("skip-l23")
|
||||
return 0
|
||||
return 1
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,681 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic.
|
||||
|
||||
Bootstrap mode (``bootstrap-upload-steps.yml``):
|
||||
- Hook uploads the entry YAML (``pipeline.yml`` / ``pipeline-npu.yml``) with one step that runs
|
||||
``upload_pipeline.py --upload <platform>/bootstrap-upload-steps.yml``.
|
||||
- Injects ``if`` by step ``key`` from skip-ci and uploads child steps (image build, L2–L5 upload).
|
||||
- Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff.
|
||||
- When only CI level YAML changes, enable **L2/L3** upload steps for affected levels only.
|
||||
|
||||
Test pipeline mode (e.g. test-merge.yml):
|
||||
- Drop steps whose ``source_file_dependencies`` do not match changed files.
|
||||
- Expand uploader-only ``mirror_hardwares`` into ``agents`` (+ optional ``image``
|
||||
for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
|
||||
- Omit ``mirror_hardwares`` to compose ``{chip}_{n}`` from pytest ``-m`` SKU
|
||||
markers plus ``cards_n`` (max of several positives; ``not cards_1`` with no
|
||||
positive ``cards_*`` uses that chip's highest existing preset). An explicit
|
||||
preset string (``h100_2``) is used as-is (marks are ignored). Names that
|
||||
are not keys in ``ci_mirror_hardwares.yml`` fail the upload.
|
||||
|
||||
Inferred chip: unset ``MIRROR_HW`` matches ``H100`` or ``L4`` in ``-m``
|
||||
(both → H100); no match skips the step. ``MIRROR_HW=b200`` matches
|
||||
``B200`` in ``-m`` (otherwise skipped). ``-m`` is not rewritten.
|
||||
``MIRROR_HW`` must be empty or ``b200``; unknown values fail the upload.
|
||||
A CUDA preset string such as ``h100_4`` is omitted when ``MIRROR_HW=b200``.
|
||||
|
||||
Usage:
|
||||
python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
|
||||
|
||||
Requires PyYAML (``pip install pyyaml``); installs it automatically when missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ModuleNotFoundError:
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "-q", "pyyaml"],
|
||||
check=True,
|
||||
)
|
||||
import yaml
|
||||
|
||||
from skip_ci import (
|
||||
ROOT,
|
||||
resolve_ci_context_from_git,
|
||||
)
|
||||
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from tests.helpers.mark import ( # noqa: E402
|
||||
get_skus_for_platform,
|
||||
get_supported_card_counts,
|
||||
)
|
||||
|
||||
# --- Constants ---
|
||||
|
||||
LOG = "upload_pipeline"
|
||||
BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml"
|
||||
BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"})
|
||||
BOOTSTRAP_UPLOAD_IF_KEYS = {
|
||||
"upload-ready-pipeline": "ready",
|
||||
"upload-merge-pipeline": "merge",
|
||||
"upload-nightly-pipeline": "nightly",
|
||||
"upload-weekly-pipeline": "weekly",
|
||||
}
|
||||
E2E_GROUP_MARKER = "E2E Test"
|
||||
CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
|
||||
|
||||
# Bootstrap Buildkite ``if`` expressions.
|
||||
# ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
|
||||
# ``*_UPLOAD_IF``: full gate for uploading that child pipeline.
|
||||
NIGHTLY_MAIN_IF = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
|
||||
NIGHTLY_LABEL_IF = (
|
||||
f"({NIGHTLY_MAIN_IF}) || "
|
||||
'(build.branch != "main" && ('
|
||||
'build.pull_request.labels includes "nightly-test" || '
|
||||
'build.pull_request.labels includes "omni-test" || '
|
||||
'build.pull_request.labels includes "tts-test" || '
|
||||
'build.pull_request.labels includes "diffusion-x2iat-test" || '
|
||||
'build.pull_request.labels includes "diffusion-x2v-test"'
|
||||
"))"
|
||||
)
|
||||
WEEKLY_E2E_IF = 'build.branch == "main" && build.env("WEEKLY") == "1"'
|
||||
WEEKLY_MAIN_IF = 'build.branch == "main" && (build.env("WEEKLY") == "1" || build.env("NON_CRITICAL") == "1")'
|
||||
WEEKLY_LABEL_IF = f'({WEEKLY_MAIN_IF}) || (build.branch != "main" && build.pull_request.labels includes "weekly-test")'
|
||||
READY_LABEL_IF = 'build.branch != "main" && build.pull_request.labels includes "ready"'
|
||||
MERGE_LABEL_IF = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
|
||||
MERGE_MAIN_IF = (
|
||||
'build.branch == "main" && build.env("NIGHTLY") != "1" && '
|
||||
'build.env("WEEKLY") != "1" && build.env("NON_CRITICAL") != "1"'
|
||||
)
|
||||
READY_UPLOAD_IF = f"({WEEKLY_E2E_IF}) || ({READY_LABEL_IF})"
|
||||
MERGE_UPLOAD_IF = f"({WEEKLY_E2E_IF}) || (({MERGE_MAIN_IF}) || ({MERGE_LABEL_IF}))"
|
||||
BOOTSTRAP_DISABLED_IF = "false"
|
||||
BOOTSTRAP_ENABLED_IF = "true"
|
||||
|
||||
|
||||
# --- Logging ---
|
||||
|
||||
|
||||
def _log(message: str) -> None:
|
||||
print(f"{LOG}: {message}", file=sys.stderr)
|
||||
|
||||
|
||||
# --- Bootstrap pipeline (bootstrap-upload-steps.yml) ---
|
||||
|
||||
|
||||
def _get_bootstrap_platform(path: Path) -> str:
|
||||
parts = path.as_posix().split("/")
|
||||
return "npu" if "npu" in parts else "cuda"
|
||||
|
||||
|
||||
def _load_bootstrap_steps(path: Path) -> str:
|
||||
if path.name != BOOTSTRAP_STEPS_FILENAME:
|
||||
raise ValueError(f"expected {BOOTSTRAP_STEPS_FILENAME}, got {path.name}")
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _format_bootstrap_if(expr: str) -> str:
|
||||
"""Return a Buildkite ``if`` string. Buildkite rejects YAML bool ``if`` values."""
|
||||
if expr in ("true", "false"):
|
||||
return expr
|
||||
return f"({expr})"
|
||||
|
||||
|
||||
def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
|
||||
if platform == "npu":
|
||||
ready_upload = READY_LABEL_IF
|
||||
merge_upload = BOOTSTRAP_DISABLED_IF
|
||||
weekly_label_if = BOOTSTRAP_DISABLED_IF
|
||||
else:
|
||||
ready_upload = READY_UPLOAD_IF
|
||||
merge_upload = MERGE_UPLOAD_IF
|
||||
weekly_label_if = WEEKLY_LABEL_IF
|
||||
|
||||
if decision.skip_all:
|
||||
# Docs / skip-mark only: no PR-label escape hatch. Main scheduled
|
||||
# NIGHTLY=1 still runs L4; WEEKLY=1 / NON_CRITICAL=1 still run L5.
|
||||
# main+WEEKLY=1 also uploads L2/L3 (those steps then pass --e2e).
|
||||
image_expr = f"({NIGHTLY_MAIN_IF}) || ({WEEKLY_MAIN_IF})" if platform == "cuda" else NIGHTLY_MAIN_IF
|
||||
ready_expr = WEEKLY_E2E_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF
|
||||
merge_expr = WEEKLY_E2E_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF
|
||||
nightly_expr = NIGHTLY_MAIN_IF
|
||||
weekly_expr = WEEKLY_MAIN_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF
|
||||
elif decision.skip_l2_l3:
|
||||
l2_enabled = decision.is_run("npu", "l2") if platform == "npu" else decision.is_run("cuda", "l2")
|
||||
l3_enabled = platform == "cuda" and decision.is_run("cuda", "l3")
|
||||
|
||||
ready_expr = ready_upload if l2_enabled else BOOTSTRAP_DISABLED_IF
|
||||
merge_expr = merge_upload if l3_enabled else BOOTSTRAP_DISABLED_IF
|
||||
nightly_expr = NIGHTLY_LABEL_IF
|
||||
weekly_expr = weekly_label_if if platform == "cuda" else BOOTSTRAP_DISABLED_IF
|
||||
|
||||
image_parts = [f"({NIGHTLY_LABEL_IF})"]
|
||||
if platform == "cuda":
|
||||
image_parts.append(f"({weekly_label_if})")
|
||||
if l2_enabled:
|
||||
image_parts.insert(0, f"({ready_upload})")
|
||||
if l3_enabled:
|
||||
image_parts.insert(1 if l2_enabled else 0, f"({merge_upload})")
|
||||
image_expr = " || ".join(image_parts)
|
||||
else:
|
||||
image_expr = BOOTSTRAP_ENABLED_IF
|
||||
ready_expr = ready_upload
|
||||
merge_expr = merge_upload if platform == "cuda" else BOOTSTRAP_DISABLED_IF
|
||||
nightly_expr = NIGHTLY_LABEL_IF
|
||||
weekly_expr = weekly_label_if if platform == "cuda" else BOOTSTRAP_DISABLED_IF
|
||||
|
||||
return {
|
||||
"image": _format_bootstrap_if(image_expr),
|
||||
"ready": _format_bootstrap_if(ready_expr),
|
||||
"merge": _format_bootstrap_if(merge_expr),
|
||||
"nightly": _format_bootstrap_if(nightly_expr),
|
||||
"weekly": _format_bootstrap_if(weekly_expr),
|
||||
}
|
||||
|
||||
|
||||
def _apply_bootstrap_if(steps: list[Any], if_exprs: dict[str, str]) -> list[Any]:
|
||||
"""Inject ``if`` by step key; drop steps that are unconditionally disabled."""
|
||||
kept: list[Any] = []
|
||||
for step in steps:
|
||||
if not isinstance(step, dict):
|
||||
kept.append(step)
|
||||
continue
|
||||
nested = step.get("steps")
|
||||
if isinstance(nested, list):
|
||||
nested_kept = _apply_bootstrap_if(nested, if_exprs)
|
||||
if nested_kept:
|
||||
kept.append({**step, "steps": nested_kept})
|
||||
continue
|
||||
key = step.get("key")
|
||||
if key in BOOTSTRAP_IMAGE_BUILD_KEYS:
|
||||
step["if"] = if_exprs["image"]
|
||||
elif key in BOOTSTRAP_UPLOAD_IF_KEYS:
|
||||
step["if"] = if_exprs[BOOTSTRAP_UPLOAD_IF_KEYS[key]]
|
||||
if_expr = step.get("if")
|
||||
if if_expr == "false":
|
||||
_log(f"omit disabled bootstrap step {key!r}")
|
||||
continue
|
||||
if if_expr == "true":
|
||||
# Unconditional step: omit ``if`` (Buildkite requires string ``if``, not YAML bool).
|
||||
step.pop("if", None)
|
||||
kept.append(step)
|
||||
return kept
|
||||
|
||||
|
||||
def _render_bootstrap_pipeline(
|
||||
steps_yaml: str,
|
||||
*,
|
||||
decision,
|
||||
path: Path,
|
||||
) -> str:
|
||||
"""Load bootstrap steps YAML and inject ``if`` expressions from skip-ci decision."""
|
||||
doc = yaml.safe_load(steps_yaml)
|
||||
if not isinstance(doc, dict):
|
||||
raise ValueError(f"invalid bootstrap steps YAML for {path}")
|
||||
steps = doc.get("steps")
|
||||
if not isinstance(steps, list):
|
||||
raise ValueError(f"bootstrap steps YAML must contain steps: list in {path}")
|
||||
|
||||
platform = _get_bootstrap_platform(path)
|
||||
if_exprs = _compute_bootstrap_if_exprs(decision=decision, platform=platform)
|
||||
doc["steps"] = _apply_bootstrap_if(steps, if_exprs)
|
||||
return yaml.safe_dump(doc, sort_keys=False)
|
||||
|
||||
|
||||
# --- Test pipeline (test-ready.yml, test-merge.yml) ---
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
|
||||
if not CI_MIRROR_HARDWARES_PATH.is_file():
|
||||
raise FileNotFoundError(f"missing CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}")
|
||||
doc = yaml.safe_load(CI_MIRROR_HARDWARES_PATH.read_text(encoding="utf-8"))
|
||||
if not isinstance(doc, dict):
|
||||
raise ValueError(f"invalid CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}")
|
||||
presets = doc.get("mirror_hardwares")
|
||||
if not isinstance(presets, dict):
|
||||
raise ValueError(f"mirror_hardwares must be a mapping in {CI_MIRROR_HARDWARES_PATH}")
|
||||
return presets
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _cuda_mirror_chips() -> tuple[str, ...]:
|
||||
"""Lowercase CUDA SKUs that have at least one ``{chip}_{n}`` preset.
|
||||
|
||||
Longer tokens first so ``h100`` is not matched as a prefix of ``h200``.
|
||||
"""
|
||||
skus = get_skus_for_platform("cuda")
|
||||
known = _load_mirror_hardwares()
|
||||
chips = {
|
||||
sku.lower() for sku in skus if any(name == sku.lower() or name.startswith(f"{sku.lower()}_") for name in known)
|
||||
}
|
||||
return tuple(sorted(chips, key=lambda chip: (-len(chip), chip)))
|
||||
|
||||
|
||||
def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
|
||||
"""Parse registered ``cards_n`` / ``not cards_n`` from a pytest ``-m`` expr.
|
||||
|
||||
Returns positive card counts and whether any ``not cards_*`` is present.
|
||||
"""
|
||||
counts = get_supported_card_counts()
|
||||
alt = "|".join(str(n) for n in sorted(counts, key=lambda n: (-len(str(n)), -n))) if counts else "(?!)"
|
||||
positives: set[int] = set()
|
||||
has_not_cards = False
|
||||
for match in re.finditer(rf"\b(?:(not)\s+)?cards_({alt})\b", expr):
|
||||
if match.group(1):
|
||||
has_not_cards = True
|
||||
else:
|
||||
positives.add(int(match.group(2)))
|
||||
return positives, has_not_cards
|
||||
|
||||
|
||||
def _read_hardware_marks(expr: str) -> set[str]:
|
||||
"""Return lowercase positive CUDA SKUs that have a mirror preset.
|
||||
|
||||
``not H100`` is ignored. SKUs with no ``{chip}_*`` preset (e.g. H200) are dropped.
|
||||
"""
|
||||
skus = sorted(get_skus_for_platform("cuda"), key=lambda s: (-len(s), s))
|
||||
if not skus:
|
||||
return set()
|
||||
alt = "|".join(re.escape(sku) for sku in skus)
|
||||
mirror = set(_cuda_mirror_chips())
|
||||
chips: set[str] = set()
|
||||
for match in re.finditer(rf"\b(?:(not)\s+)?({alt})\b", expr):
|
||||
if match.group(1):
|
||||
continue
|
||||
chip = match.group(2).lower()
|
||||
if chip in mirror:
|
||||
chips.add(chip)
|
||||
return chips
|
||||
|
||||
|
||||
# CI policy: not pytest-mark registry.
|
||||
_SUPPORTED_MIRROR_HW_SELECTORS = frozenset({"b200"})
|
||||
# Unset MIRROR_HW: match H100 then L4 in -m (skip if neither).
|
||||
_DEFAULT_INFER_CHIPS = ("h100", "l4")
|
||||
_PYTEST_MARKER_ARG = re.compile(r"-m\s+(?:\"([^\"]*)\"|'([^']*)'|(\S+))")
|
||||
_CARDS_CHIP_MAX: Literal["max"] = "max"
|
||||
|
||||
|
||||
def _get_mirror_hw_selector() -> str:
|
||||
"""Return lowercase ``MIRROR_HW``, or empty to keep string presets / marker chips.
|
||||
|
||||
A set value must be ``b200`` (case-insensitive). Unknown values fail closed
|
||||
so a typo cannot silently drop the CUDA pipeline.
|
||||
"""
|
||||
selector = os.environ.get("MIRROR_HW", "").strip().lower()
|
||||
if not selector:
|
||||
return ""
|
||||
if selector not in _SUPPORTED_MIRROR_HW_SELECTORS:
|
||||
allowed = " or ".join(repr(s) for s in sorted(_SUPPORTED_MIRROR_HW_SELECTORS))
|
||||
raise ValueError(f"unsupported MIRROR_HW={selector!r}; expected empty or {allowed}")
|
||||
return selector
|
||||
|
||||
|
||||
def _read_pytest_marks(commands: Any) -> tuple[set[str], int | Literal["max"] | None]:
|
||||
"""Read pytest ``-m`` from step commands: CUDA SKU chips and cards count.
|
||||
|
||||
*chips* are lowercase SKUs from positive markers (``not H100`` is ignored).
|
||||
*cards* is the max positive ``cards_n``, ``"max"`` when only ``not cards_*``
|
||||
is present, or ``None``.
|
||||
"""
|
||||
chunks: list[str] = []
|
||||
|
||||
def _collect(value: Any) -> None:
|
||||
if value is None:
|
||||
return
|
||||
if isinstance(value, str):
|
||||
chunks.append(value)
|
||||
elif isinstance(value, list):
|
||||
for part in value:
|
||||
_collect(part)
|
||||
else:
|
||||
chunks.append(str(value))
|
||||
|
||||
_collect(commands)
|
||||
text = "\n".join(chunks)
|
||||
|
||||
chips: set[str] = set()
|
||||
positives: set[int] = set()
|
||||
has_not_cards = False
|
||||
for match in _PYTEST_MARKER_ARG.finditer(text):
|
||||
expr = match.group(1) or match.group(2) or match.group(3) or ""
|
||||
chips |= _read_hardware_marks(expr)
|
||||
card_counts, not_cards = _read_cards_marks(expr)
|
||||
positives.update(card_counts)
|
||||
has_not_cards = has_not_cards or not_cards
|
||||
|
||||
if positives:
|
||||
cards: int | Literal["max"] | None = max(positives)
|
||||
elif has_not_cards:
|
||||
cards = _CARDS_CHIP_MAX
|
||||
else:
|
||||
cards = None
|
||||
return chips, cards
|
||||
|
||||
|
||||
def _compose_mirror_hardware_name(
|
||||
chips: set[str],
|
||||
cards: int | Literal["max"] | None,
|
||||
*,
|
||||
step_label: str,
|
||||
) -> str | None:
|
||||
"""Assemble ``{chip}_{n}`` from pytest ``-m`` chips/cards, or None to skip.
|
||||
|
||||
Unset ``MIRROR_HW``: ``H100`` if in ``-m``, else ``L4``, else skip.
|
||||
Set ``MIRROR_HW``: that chip if in ``-m``, else skip.
|
||||
``cards`` is a registered ``cards_n``, ``"max"`` for that chip's highest
|
||||
existing preset, or None (error after a chip matched — no ``cards_*``).
|
||||
"""
|
||||
selector = _get_mirror_hw_selector()
|
||||
if selector:
|
||||
chip = selector if selector in chips else None
|
||||
skip_reason = f"MIRROR_HW={selector!r} not in -m SKUs {sorted(chips)}"
|
||||
else:
|
||||
chip = next((c for c in _DEFAULT_INFER_CHIPS if c in chips), None)
|
||||
skip_reason = f"no H100 or L4 in -m SKUs {sorted(chips)}"
|
||||
if chip is None:
|
||||
_log(f"skip {step_label}: {skip_reason}")
|
||||
return None
|
||||
if cards is None:
|
||||
raise ValueError(
|
||||
f"step {step_label!r} has a pytest -m SKU marker but no cards_* "
|
||||
f"(or not cards_*); add cards_n or set mirror_hardwares to force a preset",
|
||||
)
|
||||
|
||||
count_source = "not cards_*" if cards == _CARDS_CHIP_MAX else f"cards_{cards}"
|
||||
count = None if cards == _CARDS_CHIP_MAX else cards
|
||||
registered = get_supported_card_counts()
|
||||
if count is not None and count not in registered:
|
||||
supported = ", ".join(f"cards_{n}" for n in sorted(registered)) or "(none)"
|
||||
raise ValueError(
|
||||
f"{count_source} in step {step_label!r} is not a registered cards_* mark; supported: {supported}",
|
||||
)
|
||||
known = _load_mirror_hardwares()
|
||||
if count is None:
|
||||
available = [n for n in sorted(registered) if f"{chip}_{n}" in known]
|
||||
if not available:
|
||||
raise ValueError(
|
||||
f"{count_source} in step {step_label!r} has no CUDA count presets for {chip}",
|
||||
)
|
||||
count = max(available)
|
||||
chosen = f"{chip}_{count}"
|
||||
if chosen not in known:
|
||||
raise ValueError(
|
||||
f"{count_source} in step {step_label!r} has no preset {chosen!r}",
|
||||
)
|
||||
_log(f"{step_label}: {count_source} → {chosen!r}")
|
||||
return chosen
|
||||
|
||||
|
||||
def _resolve_mirror_hardware_name(hardware: Any, *, step_label: str) -> str | None:
|
||||
"""Map ``mirror_hardwares`` to a ``ci_mirror_hardwares.yml`` preset, or None to skip.
|
||||
|
||||
The value is a preset name (``h100_2``). Unknown names fail later when the
|
||||
registry is loaded. CUDA names are omitted when ``MIRROR_HW`` names a
|
||||
different chip; NPU strings ignore ``MIRROR_HW``.
|
||||
"""
|
||||
if isinstance(hardware, bool):
|
||||
raise ValueError(f"mirror_hardwares must not be a boolean in step {step_label!r}")
|
||||
if isinstance(hardware, str):
|
||||
name = hardware.strip()
|
||||
elif isinstance(hardware, int):
|
||||
name = str(hardware)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"mirror_hardwares must be a preset string in step {step_label!r}",
|
||||
)
|
||||
if not name:
|
||||
raise ValueError(f"mirror_hardwares must be a non-empty string in step {step_label!r}")
|
||||
|
||||
selector = _get_mirror_hw_selector()
|
||||
token = name.lower()
|
||||
chip = next(
|
||||
(c for c in _cuda_mirror_chips() if token == c or token.startswith(f"{c}_")),
|
||||
None,
|
||||
)
|
||||
if selector and chip is not None and chip != selector:
|
||||
_log(
|
||||
f"skip {step_label}: preset {name!r} is {chip} hardware; MIRROR_HW={selector!r}",
|
||||
)
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Replace ``mirror_hardwares`` (or infer it from ``-m``) with preset agents/plugins.
|
||||
|
||||
Explicit preset string: used as-is; pytest marks are ignored.
|
||||
Omitted key: match H100/L4 (or ``MIRROR_HW``) in ``-m``, then compose
|
||||
``{chip}_{n}`` from ``cards_n``. No match skips the step. Steps with no
|
||||
SKU/cards and no ``mirror_hardwares`` are left unchanged (CPU jobs).
|
||||
|
||||
Returns None when the current ``MIRROR_HW`` selector does not apply.
|
||||
"""
|
||||
step_label = _get_step_label(step)
|
||||
has_pool = step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None
|
||||
|
||||
if "mirror_hardwares" not in step:
|
||||
if has_pool:
|
||||
return step
|
||||
chips, cards = _read_pytest_marks(step.get("commands"))
|
||||
if not chips and cards is None:
|
||||
return step
|
||||
preset_name = _compose_mirror_hardware_name(chips, cards, step_label=step_label)
|
||||
else:
|
||||
if has_pool:
|
||||
raise ValueError(
|
||||
f"step {step_label!r} sets mirror_hardwares together with agents/plugins/image; "
|
||||
f"use mirror_hardwares only",
|
||||
)
|
||||
preset_name = _resolve_mirror_hardware_name(
|
||||
step.get("mirror_hardwares"),
|
||||
step_label=step_label,
|
||||
)
|
||||
|
||||
if preset_name is None:
|
||||
return None
|
||||
|
||||
preset = _load_mirror_hardwares().get(preset_name)
|
||||
if preset is None:
|
||||
known = ", ".join(sorted(_load_mirror_hardwares()))
|
||||
raise ValueError(
|
||||
f"unknown mirror_hardwares {preset_name!r} in step {step_label!r}; known: {known}",
|
||||
)
|
||||
|
||||
expanded = copy.deepcopy(preset)
|
||||
merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
|
||||
# Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
|
||||
if "retry" in step:
|
||||
merged["retry"] = step["retry"]
|
||||
return merged
|
||||
|
||||
|
||||
def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool:
|
||||
for path in changed_files:
|
||||
for prefix in prefixes:
|
||||
normalized = prefix.rstrip("/")
|
||||
if path == normalized or path.startswith(f"{normalized}/"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _get_step_label(step: dict[str, Any]) -> str:
|
||||
return str(step.get("group") or step.get("label") or "<step>")
|
||||
|
||||
|
||||
def _process_test_steps(
|
||||
steps: list[Any],
|
||||
changed_files: list[str] | None,
|
||||
) -> list[Any]:
|
||||
"""Drop steps by ``source_file_dependencies`` when *changed_files* is set; always strip that field."""
|
||||
processed: list[Any] = []
|
||||
for step in steps:
|
||||
if not isinstance(step, dict):
|
||||
processed.append(step)
|
||||
continue
|
||||
|
||||
deps = step.get("source_file_dependencies")
|
||||
if deps is not None and not isinstance(deps, list):
|
||||
raise ValueError(
|
||||
f"source_file_dependencies must be a list in step {_get_step_label(step)!r}",
|
||||
)
|
||||
if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):
|
||||
_log(f"skip {_get_step_label(step)!r} (no changes under {deps})")
|
||||
continue
|
||||
|
||||
nested = step.get("steps")
|
||||
if nested is not None:
|
||||
kept_nested = _process_test_steps(nested, changed_files)
|
||||
if not kept_nested:
|
||||
_log(f"omit empty group {_get_step_label(step)!r}")
|
||||
continue
|
||||
new_step = {key: value for key, value in step.items() if key != "source_file_dependencies"}
|
||||
new_step["steps"] = kept_nested
|
||||
processed.append(new_step)
|
||||
continue
|
||||
|
||||
leaf = {key: value for key, value in step.items() if key != "source_file_dependencies"}
|
||||
expanded = _expand_mirror_hardwares(leaf)
|
||||
if expanded is None:
|
||||
continue
|
||||
processed.append(expanded)
|
||||
|
||||
return processed
|
||||
|
||||
|
||||
def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
|
||||
"""Keep only top-level groups whose name contains ``E2E_GROUP_MARKER``."""
|
||||
selected = [
|
||||
step
|
||||
for step in steps
|
||||
if isinstance(step, dict) and isinstance(step.get("group"), str) and E2E_GROUP_MARKER in step["group"]
|
||||
]
|
||||
if not selected:
|
||||
_log(f"no group matching {E2E_GROUP_MARKER!r} found")
|
||||
else:
|
||||
_log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}")
|
||||
return selected
|
||||
|
||||
|
||||
def _render_test_pipeline(
|
||||
doc: dict[str, Any],
|
||||
changed_files: list[str] | None,
|
||||
*,
|
||||
e2e_only: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
|
||||
# Validate MIRROR_HW once up front. Per-step skip must not run for typos
|
||||
# (e.g. b20o), or a pipeline can shrink to leftover CPU-only steps and
|
||||
# still upload successfully.
|
||||
_get_mirror_hw_selector()
|
||||
steps = doc.get("steps")
|
||||
if not isinstance(steps, list):
|
||||
return doc
|
||||
if e2e_only:
|
||||
steps = _select_e2e_group_steps(steps)
|
||||
steps = _process_test_steps(steps, changed_files)
|
||||
return {**doc, "steps": steps}
|
||||
|
||||
|
||||
# --- Entry (read file → bootstrap or test render → YAML string) ---
|
||||
|
||||
|
||||
def _render_pipeline(
|
||||
path: Path,
|
||||
*,
|
||||
force_all: bool = False,
|
||||
e2e_only: bool = False,
|
||||
) -> str:
|
||||
if path.name == BOOTSTRAP_STEPS_FILENAME:
|
||||
ctx = resolve_ci_context_from_git()
|
||||
continuation = _load_bootstrap_steps(path)
|
||||
return _render_bootstrap_pipeline(
|
||||
continuation,
|
||||
decision=ctx.decision,
|
||||
path=path,
|
||||
)
|
||||
|
||||
text = path.read_text(encoding="utf-8")
|
||||
ctx = resolve_ci_context_from_git()
|
||||
changed_files = None if force_all or e2e_only else ctx.changed_files
|
||||
|
||||
doc = yaml.safe_load(text)
|
||||
if not isinstance(doc, dict):
|
||||
raise ValueError(f"invalid pipeline YAML: {path}")
|
||||
|
||||
doc = _render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
|
||||
return yaml.safe_dump(doc, sort_keys=False)
|
||||
|
||||
|
||||
def _upload_to_buildkite(content: str) -> None:
|
||||
subprocess.run(
|
||||
["buildkite-agent", "pipeline", "upload"],
|
||||
input=content,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
# --- CLI ---
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"pipeline",
|
||||
nargs="?",
|
||||
default=".buildkite/cuda/bootstrap-upload-steps.yml",
|
||||
help="Pipeline YAML path (default: .buildkite/cuda/bootstrap-upload-steps.yml)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--upload",
|
||||
action="store_true",
|
||||
help="Pipe rendered YAML to buildkite-agent pipeline upload",
|
||||
)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="Keep all steps (disable diff-aware skipping)",
|
||||
)
|
||||
mode.add_argument(
|
||||
"--e2e",
|
||||
action="store_true",
|
||||
help="Keep only the E2E Test group",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
path = Path(args.pipeline)
|
||||
if not path.is_absolute():
|
||||
path = ROOT / path
|
||||
if not path.is_file():
|
||||
_log(f"missing pipeline file: {path}")
|
||||
return 1
|
||||
|
||||
rendered = _render_pipeline(path, force_all=args.all, e2e_only=args.e2e)
|
||||
if args.upload:
|
||||
_upload_to_buildkite(rendered)
|
||||
else:
|
||||
sys.stdout.write(rendered)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,55 +0,0 @@
|
||||
# Uploaded by upload_pipeline.py --upload (not the Buildkite hook).
|
||||
# ``if`` is injected by step ``key`` from skip-ci at upload time.
|
||||
steps:
|
||||
- label: ":docker: Build image"
|
||||
key: image-build
|
||||
commands:
|
||||
- "bash .buildkite/cuda/scripts/build-ci-image.sh"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L2 Test — PR ready label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group)
|
||||
- label: "Upload Ready Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${WEEKLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L3 Test — PR merge-test label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group)
|
||||
- label: "Upload Merge Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-merge-pipeline
|
||||
commands:
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${WEEKLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
- label: "Upload Nightly Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-nightly-pipeline
|
||||
commands:
|
||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-nightly.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (scheduled), or PR with weekly-test label
|
||||
- label: "Upload Weekly Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-weekly-pipeline
|
||||
commands:
|
||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -1,9 +0,0 @@
|
||||
# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml
|
||||
# and injects ``if`` by step ``key`` from skip-ci.
|
||||
steps:
|
||||
- label: ":github: Resolve skip-ci & upload pipeline"
|
||||
key: upload-ci-pipeline
|
||||
commands:
|
||||
- "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -1,35 +0,0 @@
|
||||
steps:
|
||||
- label: ":docker: Build image"
|
||||
key: image-build
|
||||
commands:
|
||||
- "bash .buildkite/cuda/scripts/build-ci-image.sh"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L2 Test
|
||||
- label: "Upload Ready Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-ready-pipeline
|
||||
commands:
|
||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L3 Test
|
||||
- label: "Upload Merge Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-merge-pipeline
|
||||
commands:
|
||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L4 Test
|
||||
- label: "Upload Nightly Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
commands:
|
||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Build and push the CI Docker image with registry-based layer caching.
|
||||
#
|
||||
# Usage: build-ci-image.sh
|
||||
#
|
||||
# Expects the following environment variables (set by Buildkite):
|
||||
# BUILDKITE_COMMIT - commit SHA used as the image tag
|
||||
ECR_NAMESPACE="public.ecr.aws/q9t5s3a7"
|
||||
REGISTRY="${ECR_NAMESPACE}/vllm-ci-test-repo"
|
||||
REGION="us-east-1"
|
||||
DOCKERFILE="docker/Dockerfile.ci"
|
||||
BUILDER_NAME="vllm-omni-builder"
|
||||
|
||||
# Ensure that the env vars are actually set, otherwise exit early
|
||||
if [ -z "${BUILDKITE_COMMIT:-}" ]; then
|
||||
echo "ERROR: BUILDKITE_COMMIT is not set"
|
||||
exit 1
|
||||
fi
|
||||
echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
|
||||
|
||||
# Authenticate to ECR Public
|
||||
aws ecr-public get-login-password --region "$REGION" \
|
||||
| docker login --username AWS --password-stdin "$ECR_NAMESPACE"
|
||||
|
||||
# Compute the cache tag for the dependencies image as the hash of the contents.
|
||||
# This is because dependencies don't change often, so most PRs will have the same hash.
|
||||
#
|
||||
# The Dockerfile is also included to avoid churn in the cached image based on the vLLM version.
|
||||
DEP_FILES="pyproject.toml setup.py requirements/*.txt docker/Dockerfile.ci"
|
||||
CACHE_KEY=$(cat ${DEP_FILES} | sha256sum | cut -c1-16)
|
||||
CACHE_TAG="deps-cache-${CACHE_KEY}"
|
||||
echo "Cache key: ${CACHE_TAG}"
|
||||
|
||||
# Set up buildx with docker-container driver; we need to do this
|
||||
# since cache export is not supported for the default docker driver
|
||||
# that is running in the CI.
|
||||
docker buildx inspect $BUILDER_NAME >/dev/null 2>&1 \
|
||||
|| docker buildx create --name $BUILDER_NAME --driver docker-container
|
||||
docker buildx use $BUILDER_NAME
|
||||
|
||||
echo "Building image tag: ${REGISTRY}:${BUILDKITE_COMMIT}"
|
||||
|
||||
# Build + push the image to the registry, using the hash from the dependencies.
|
||||
# Note that the caching here is layerwise, so we're just making sure we always
|
||||
# can cache hit on everything but the last copy at the moment.
|
||||
docker buildx build --push --progress=plain \
|
||||
--cache-from "type=registry,ref=${REGISTRY}:${CACHE_TAG}" \
|
||||
--cache-to "type=registry,ref=${REGISTRY}:${CACHE_TAG},mode=max,compression=zstd" \
|
||||
--file "$DOCKERFILE" \
|
||||
-t "${REGISTRY}:${BUILDKITE_COMMIT}" .
|
||||
@@ -1,277 +0,0 @@
|
||||
# CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
|
||||
# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main.
|
||||
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
steps:
|
||||
- group: ":card_index_dividers: Simple Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Simple · Model Executor Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- "pytest -sv tests/model_executor -m 'core_model and cpu'"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Simple · Diffusion Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- "pytest -sv tests/diffusion -m 'core_model and cpu'"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Simple · Engine&Entrypoints Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Simple · Other Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Diffusion · LoRA Test"
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Distributed Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- |
|
||||
timeout 55m bash -c "
|
||||
set -e
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- group: ":card_index_dividers: Entrypoints Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Entrypoint Test with H100"
|
||||
# TEMPORARY 60 -> 90: engine startup/sleep-wake is ~13-15 min slower per
|
||||
# run on the vLLM 0.26 image (model loading ~2x, init ~2x; see builds
|
||||
# 2915 vs 2916), which pushed this suite past 60 min. Revert once the
|
||||
# 0.26 startup regression is understood and fixed.
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Entrypoint Test with L4"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
- tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/models/common/qwen3_code_predictor.py
|
||||
- vllm_omni/model_executor/models/common/snake_activation.py
|
||||
- vllm_omni/model_executor/models/common/alias_free_activation.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
- vllm_omni/entrypoints/openai/serving_speech.py
|
||||
- vllm_omni/entrypoints/openai/tts_adapters/
|
||||
- vllm_omni/engine/stage_pool.py
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Qwen3-TTS Base Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
- tests/e2e/offline_inference/test_qwen3_tts_base.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/models/common/qwen3_code_predictor.py
|
||||
- vllm_omni/model_executor/models/common/snake_activation.py
|
||||
- vllm_omni/model_executor/models/common/alias_free_activation.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
- vllm_omni/entrypoints/openai/serving_speech.py
|
||||
- vllm_omni/entrypoints/openai/tts_adapters/
|
||||
- vllm_omni/engine/stage_pool.py
|
||||
- .buildkite/common/scripts/run_cov_split.sh
|
||||
- pyproject.toml
|
||||
# 40 -> 60: the coverage split runs the model twice.
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- |
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
.buildkite/common/scripts/run_cov_split.sh \
|
||||
--model-id qwen3_tts \
|
||||
--offline tests/e2e/offline_inference/test_qwen3_tts_base.py \
|
||||
--online tests/e2e/online_serving/test_qwen3_tts_base.py \
|
||||
-- -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
- tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
|
||||
- tests/e2e/online_serving/test_qwen3_omni.py
|
||||
- tests/helpers/runtime.py
|
||||
- vllm_omni/entrypoints/openai/serving_chat.py
|
||||
- vllm_omni/model_executor/models/qwen3_omni/
|
||||
- vllm_omni/model_executor/models/common/qwen3_code_predictor.py
|
||||
- vllm_omni/model_executor/models/common/snake_activation.py
|
||||
- vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
- vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
- vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_minicpmo_4_5.py
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5.py
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/utils.py
|
||||
- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# h100_2, not h100_1: advanced_model asserts transcribe every audio
|
||||
# response with Whisper, and tests/helpers/media.py only uses an
|
||||
# accelerator for Whisper when a spare GPU exists. On a single card it
|
||||
# grades on CPU: ~8-10 min per clip vs ~30 s, which alone overruns the
|
||||
# 50-minute job timeout (build 2956).
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
source_file_dependencies:
|
||||
- tests/assets/minicpmo_4_5/response_required_16k.wav
|
||||
- tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
||||
- tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
- tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
|
||||
- vllm_omni/experimental/fullduplex/
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/utils.py
|
||||
- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
|
||||
- vllm_omni/core/sched/
|
||||
- vllm_omni/outputs/multimodal_accumulation.py
|
||||
- vllm_omni/engine/stage_init_utils.py
|
||||
- vllm_omni/engine/async_omni_engine.py
|
||||
- vllm_omni/engine/orchestrator.py
|
||||
- vllm_omni/entrypoints/cli/serve.py
|
||||
- vllm_omni/entrypoints/openai/api_server.py
|
||||
- vllm_omni/entrypoints/openai/duplex_capability.py
|
||||
- vllm_omni/config/stage_config.py
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · Qwen Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
- vllm_omni/diffusion/models/dmd2/
|
||||
- vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c '
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
'
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_wan22_t2v.py
|
||||
- tests/e2e/online_serving/test_wan22_t2v.py
|
||||
- vllm_omni/diffusion/models/wan2_2/
|
||||
- vllm_omni/diffusion/models/dmd2/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · HunyuanVideo-1.5 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
|
||||
- vllm_omni/diffusion/models/hunyuan_video/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/minimax_h3/
|
||||
- tests/helpers/assertions.py
|
||||
- tests/helpers/mark.py
|
||||
- tests/helpers/media.py
|
||||
- tests/helpers/runtime.py
|
||||
- vllm_omni/diffusion/models/minimax_h3/
|
||||
- vllm_omni/diffusion/lora/
|
||||
- vllm_omni/diffusion/offloader/
|
||||
- vllm_omni/diffusion/distributed/
|
||||
- vllm_omni/entrypoints/openai/
|
||||
# Keep four server lifecycles in one Buildkite job, but never in one
|
||||
# pytest process. DLO uses multiprocessing shared-memory queues; the
|
||||
# parent's resource_tracker can retain a stale /psm_* entry during
|
||||
# teardown and strand the next Ref2VA collective.
|
||||
# A per-file process plus a hard timeout also makes a regression fail
|
||||
# instead of leaving the merge queue stuck indefinitely.
|
||||
timeout_in_minutes: 150
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
if [ -f vllm_omni/diffusion/models/minimax_h3/lora.py ]; then
|
||||
export VLLM_TEST_MINIMAX_H3_TURBO_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="lightx2v/Minimax-h3-Turbo", filename="minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"))')
|
||||
fi
|
||||
if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then
|
||||
export VLLM_TEST_MINIMAX_H3_FASTH3_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA", filename="dense-datafree/adapter_model.safetensors"))')
|
||||
fi
|
||||
timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_fl2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_ref2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_2
|
||||
@@ -1,534 +0,0 @@
|
||||
# CUDA CI L4 (`full_model`): nightly and release share this file.
|
||||
# Uploaded by Upload Nightly Pipeline: NIGHTLY=1, or PR labels (nightly-test / omni-test / ...).
|
||||
# Hardware: omit mirror_hardwares to compose from -m SKU + cards_n; or set a preset string to force.
|
||||
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
steps:
|
||||
# Group: collapses under one heading in the Buildkite UI; child steps still run in parallel.
|
||||
- group: ":card_index_dividers: Omni Model Test"
|
||||
key: nightly-omni-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
|
||||
steps:
|
||||
|
||||
- label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
|
||||
- label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export SEED_TTS_WER_EVAL=1
|
||||
- export SEED_TTS_EVAL_DEVICE=cuda:1
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export SEED_TTS_WER_EVAL=1
|
||||
- export SEED_TTS_EVAL_DEVICE=cuda:1
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
||||
key: nightly-omni-performance-no-async-chunk
|
||||
timeout_in_minutes: 300
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · Perf Test · Async Chunk"
|
||||
key: nightly-omni-performance-async-chunk
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and full_model"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
|
||||
timeout_in_minutes: 45
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: nightly-tts-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
steps:
|
||||
|
||||
- label: ":full_moon: TTS · Function Test with L4"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: ":full_moon: TTS · Perf Test · Single-GPU"
|
||||
key: nightly-tts-performance-single
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: TTS · Perf Test · 2-GPU"
|
||||
key: nightly-tts-performance-2gpu
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
|
||||
key: nightly-tiny-model-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test"
|
||||
steps:
|
||||
|
||||
- label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU"
|
||||
timeout_in_minutes: 30
|
||||
# NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
|
||||
# so nightly only adds the multi-GPU parallelism configurations.
|
||||
#
|
||||
# This is a bit confusing, but `full_model` is used to mark the multigpu tests, and
|
||||
# --run-level `core_model` is used to indicate that we should use tiny model weights
|
||||
# for the CI, since these tests should pass irrespective of whether or not we use
|
||||
# the full weights or not.
|
||||
commands:
|
||||
- pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model"
|
||||
mirror_hardwares: l4_2
|
||||
|
||||
- label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU"
|
||||
timeout_in_minutes: 30
|
||||
# CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count).
|
||||
commands:
|
||||
- pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
|
||||
key: nightly-diffusion-x2iat-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2iat-test"
|
||||
steps:
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_qwen_image_expansion.py
|
||||
-m "full_model and diffusion and H100 and cards_1"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_qwen_image_expansion.py
|
||||
-m "full_model and diffusion and H100 and cards_2"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Boogu Edit CFG · 3-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_boogu_image_edit_expansion.py
|
||||
-k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
|
||||
-m "slow and diffusion and cuda and H100 and cards_3"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_3
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · 4-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and cards_4" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
|
||||
- pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
|
||||
- export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
|
||||
- pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: ":full_moon: HiDream-O1-Image · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-single
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- group: ":card_index_dividers: Diffusion X2V Model Test"
|
||||
key: nightly-diffusion-x2v-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2v-test"
|
||||
steps:
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Doc Test"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Other Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Other Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
|
||||
key: nightly-diffusion-x2v-performance-single
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
|
||||
key: nightly-diffusion-x2v-performance-2gpu
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2"
|
||||
EXIT1=$$?
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2"
|
||||
EXIT2=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
|
||||
exit $$EXIT2
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
|
||||
key: nightly-diffusion-x2v-performance-hunyuanvideo15
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
|
||||
EXIT1=$$?
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
|
||||
EXIT2=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
|
||||
exit $$EXIT2
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
|
||||
key: nightly-diffusion-x2v-performance-minimax-h3
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
key: nightly-diffusion-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
||||
steps:
|
||||
- label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model"
|
||||
mirror_hardwares: l4_2
|
||||
|
||||
- label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model"
|
||||
mirror_hardwares: l4_3
|
||||
|
||||
- label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model"
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: ":full_moon: Diffusion · Offloader Test"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/offloader -m "full_model and cuda"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion · Quantization Test with L4"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pip install "vllm-gguf-plugin==0.0.4"
|
||||
- pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
|
||||
# Do not add new depends_on entries or extend this step.
|
||||
- label: ":email: Nightly Collection & Email"
|
||||
key: nightly-perf-distribution
|
||||
depends_on:
|
||||
- nightly-omni-performance-no-async-chunk
|
||||
- nightly-omni-performance-async-chunk
|
||||
- nightly-tts-performance-single
|
||||
- nightly-tts-performance-2gpu
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-single
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-4gpu
|
||||
- nightly-diffusion-x2v-performance-single
|
||||
- nightly-diffusion-x2v-performance-2gpu
|
||||
if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
|
||||
commands:
|
||||
- pip install openpyxl
|
||||
- export DEFAULT_INPUT_DIR=tests/dfx/perf/results
|
||||
- export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-single
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-2gpu
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-single
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-4gpu
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-single
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-2gpu
|
||||
- python tools/nightly/generate_nightly_perf_excel.py
|
||||
- python tools/nightly/generate_nightly_perf_html.py
|
||||
- python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
|
||||
- buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx"
|
||||
- buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -1,342 +0,0 @@
|
||||
# CUDA CI L2 (`core_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
|
||||
# Uploaded by Upload Ready Pipeline: PR label `ready`, or main+WEEKLY=1 with --e2e.
|
||||
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
steps:
|
||||
- group: ":card_index_dividers: Simple Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: "Simple · Model Executor Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- "pytest -sv tests/model_executor -m 'core_model and cpu'"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Simple · Diffusion Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- "pytest -sv tests/diffusion -m 'core_model and cpu'"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Simple · Engine&Entrypoints Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Simple · Other Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
|
||||
- label: "Diffusion · Offloader Test"
|
||||
commands:
|
||||
- timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Cache Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Batch Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Distributed Test · Single-GPU"
|
||||
commands:
|
||||
- timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Distributed Test · 2-GPU"
|
||||
commands:
|
||||
- timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
|
||||
mirror_hardwares: l4_2
|
||||
|
||||
- label: "Diffusion · Distributed & Attention Test · 4-GPU"
|
||||
commands:
|
||||
- timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
|
||||
- timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: "Diffusion · Model Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Other Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
|
||||
- label: "Tiny Model Tests · Diffusion (base)"
|
||||
commands:
|
||||
- timeout 15m pytest -sv -n 4 tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Engine&Model Executor Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 15m bash -c "
|
||||
pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Distributed Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Entrypoints Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: "Custom Pipeline Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/features/custom_pipeline/
|
||||
- tests/e2e/features/helpers/custom_pipeline.py
|
||||
- vllm_omni/diffusion/
|
||||
- vllm_omni/engine/
|
||||
- vllm_omni/entrypoints/
|
||||
- vllm_omni/inputs/
|
||||
- vllm_omni/outputs/
|
||||
commands:
|
||||
- timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
|
||||
- label: "RLHF · VeRL-Omni E2E Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
|
||||
- tests/e2e/features/helpers/verl_omni_server.py
|
||||
- tests/e2e/features/helpers/custom_pipeline.py
|
||||
- vllm_omni/diffusion/
|
||||
- vllm_omni/engine/
|
||||
- vllm_omni/entrypoints/
|
||||
- vllm_omni/inputs/
|
||||
- vllm_omni/outputs/
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_omni.py
|
||||
- tests/helpers/runtime.py
|
||||
- vllm_omni/entrypoints/openai/serving_chat.py
|
||||
- vllm_omni/model_executor/models/qwen3_omni/
|
||||
- vllm_omni/model_executor/models/common/qwen3_code_predictor.py
|
||||
- vllm_omni/model_executor/models/common/snake_activation.py
|
||||
- vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
- vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Offline Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_minicpmo_4_5.py
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/utils.py
|
||||
- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Online Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5.py
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/utils.py
|
||||
- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
source_file_dependencies:
|
||||
- tests/assets/minicpmo_4_5/response_required_16k.wav
|
||||
- tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
||||
- tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
- tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
|
||||
- vllm_omni/experimental/fullduplex/
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/utils.py
|
||||
- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
|
||||
- vllm_omni/core/sched/
|
||||
- vllm_omni/outputs/multimodal_accumulation.py
|
||||
- vllm_omni/engine/stage_init_utils.py
|
||||
- vllm_omni/engine/async_omni_engine.py
|
||||
- vllm_omni/engine/orchestrator.py
|
||||
- vllm_omni/entrypoints/cli/serve.py
|
||||
- vllm_omni/entrypoints/openai/api_server.py
|
||||
- vllm_omni/entrypoints/openai/duplex_capability.py
|
||||
- vllm_omni/config/stage_config.py
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Perf Test"
|
||||
timeout_in_minutes: 60
|
||||
source_file_dependencies:
|
||||
- tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/models/common/qwen3_code_predictor.py
|
||||
- vllm_omni/model_executor/models/common/snake_activation.py
|
||||
- vllm_omni/model_executor/models/common/alias_free_activation.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
- vllm_omni/entrypoints/openai/serving_speech.py
|
||||
- vllm_omni/entrypoints/openai/tts_adapters/
|
||||
- vllm_omni/engine/stage_pool.py
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_wan22_t2v.py
|
||||
- vllm_omni/diffusion/models/wan2_2/
|
||||
- vllm_omni/diffusion/models/dmd2/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · Cosmos3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_cosmos3.py
|
||||
- vllm_omni/diffusion/models/cosmos3/
|
||||
- vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
|
||||
- vllm_omni/experimental/world_models/
|
||||
- vllm_omni/model_extras/cosmos3.py
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · MiniMax-H3 DLO DP2 T2VA Test"
|
||||
timeout_in_minutes: 90
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
|
||||
- tests/helpers/assertions.py
|
||||
- tests/helpers/runtime.py
|
||||
- vllm_omni/diffusion/models/minimax_h3/
|
||||
- vllm_omni/diffusion/offloader/
|
||||
- vllm_omni/diffusion/distributed/
|
||||
- vllm_omni/entrypoints/openai/
|
||||
commands:
|
||||
- |
|
||||
timeout 80m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py -m 'core_model and cuda and diffusion' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Diffusion · Qwen Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
- vllm_omni/diffusion/models/dmd2/
|
||||
- vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · HunyuanVideo-1.5 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
|
||||
- vllm_omni/diffusion/models/hunyuan_video/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
@@ -1,250 +0,0 @@
|
||||
# CUDA CI L5 (`slow` / reliability / extra coverage).
|
||||
# Uploaded by Upload Weekly Pipeline: WEEKLY=1 / NON_CRITICAL=1, or PR label `weekly-test`.
|
||||
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
steps:
|
||||
- label: "Simple · CPU Coverage Test"
|
||||
key: weekly-simple-cpu-coverage
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("WEEKLY") == "1"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- |
|
||||
set +e
|
||||
REPORT="coverage-core-model-cpu-$${BUILDKITE_STEP_ID:-local}.xml"
|
||||
pytest -sv tests/ -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:$${REPORT}
|
||||
EXIT=$$?
|
||||
gzip -9 -f "$${REPORT}"
|
||||
buildkite-agent artifact upload "$${REPORT}.gz"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: "Reliability Test"
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
|
||||
steps:
|
||||
|
||||
- label: "Reliability Test · qwen3-omni"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Reliability Test · wan22"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Reliability Test · hunyuan-image"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: "Reliability Test · Invalid parameters · H100 · Single-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Reliability Test · Invalid parameters · H100 · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Reliability Test · Invalid parameters · L4"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: "Perf Test"
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
|
||||
steps:
|
||||
|
||||
- label: ":full_moon: Omni · Perf Test · vLLM Text"
|
||||
key: weekly-omni-performance-vllm-text
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · Perf Test · Async Chunk · Random"
|
||||
key: weekly-omni-performance-async-chunk-random
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and slow"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · Perf Test · Multi-Replica"
|
||||
key: weekly-omni-performance-multi-replicas
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_3
|
||||
|
||||
- group: ":card_index_dividers: E2E Tests"
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("NON_CRITICAL") == "1"
|
||||
steps:
|
||||
|
||||
# Online Qwen2.5 expansion is 2-card; offline expansion + AutoRound are 4-card.
|
||||
- label: "Omni · L4 · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_2" --run-level "core_model"
|
||||
mirror_hardwares: l4_2
|
||||
|
||||
- label: "Omni · L4 · 4-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_4" --run-level "core_model"
|
||||
- >-
|
||||
pytest -s -v tests/e2e/
|
||||
--ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
|
||||
--ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
|
||||
-m "slow and L4 and omni and cards_4"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: "Omni · H100 · Single-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and H100 and omni and cards_1"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · H100 · 4-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and H100 and omni and cards_4"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: "TTS · L4"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
failed=0
|
||||
for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
|
||||
pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
|
||||
done
|
||||
exit $$failed
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · H100"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and H100 and tts"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# Own step: the image has no NanoCodec, so this file import-skips out of
|
||||
# the mark-selected TTS steps above. The overrides keep nemo_toolkit[tts]
|
||||
# from downgrading transformers; see
|
||||
# examples/offline_inference/text_to_speech/README.md.
|
||||
- label: "TTS · Gepard-1.0 · L4"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt
|
||||
- uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt
|
||||
- >-
|
||||
pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py
|
||||
-m "tts and cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · L4 · Single-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and L4 and cards_1"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · L4 · 2-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and L4 and cards_2"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_2
|
||||
|
||||
- label: "Diffusion · L4 · 4-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and L4 and cards_4"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: "Diffusion · H100 · Single-GPU"
|
||||
timeout_in_minutes: 150
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and cards_1"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · H100 · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and cards_2"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Diffusion · H100 · 4-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and cards_4"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_4
|
||||
@@ -1,16 +0,0 @@
|
||||
# .buildkite/intel/pipeline-intel.yml
|
||||
steps:
|
||||
- group: "INTEL Tests"
|
||||
steps:
|
||||
- label: "Intel: XPU Test"
|
||||
command: "bash .buildkite/intel/scripts/run-xpu-test.sh"
|
||||
agents:
|
||||
queue: "intel-gpu-omni"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
# Buildkite will automatically replace this with the actual commit hash
|
||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
||||
VLLM_VERSION: "v0.28.0"
|
||||
priority: 100
|
||||
timeout_in_minutes: 60
|
||||
soft_fail: false
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
# vllm-omni Intel bootstrap
|
||||
# Uses static pipeline-intel.yml for Intel XPU tests
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/common/scripts/resolve_skip_ci.sh
|
||||
|
||||
upload_pipeline() {
|
||||
echo "--- 🛠 Preparing Intel pipeline"
|
||||
buildkite-agent pipeline upload .buildkite/intel/pipeline-intel.yml
|
||||
}
|
||||
|
||||
gate_bootstrap_ci intel l2
|
||||
|
||||
upload_pipeline
|
||||
@@ -1,127 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script build the XPU docker image and run the offline inference inside the container.
|
||||
set -ex
|
||||
|
||||
omni_source_dir=$(git rev-parse --show-toplevel)
|
||||
|
||||
: "${VLLM_VERSION:?VLLM_VERSION must be set}"
|
||||
# Official upstream XPU image. It is published from vllm's own
|
||||
# docker/Dockerfile.xpu, which is exactly what build_vllm_base() clones and
|
||||
# builds below, so pulling it is equivalent to building the fallback base.
|
||||
upstream_base_image="vllm/vllm-openai-xpu:${VLLM_VERSION}"
|
||||
local_base_image="xpu/vllm-omni-ci-base:${VLLM_VERSION}"
|
||||
image_name="xpu/vllm-omni-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT must be set}"
|
||||
container_name="xpu_${BUILDKITE_COMMIT}_$(
|
||||
tr -dc A-Za-z0-9 </dev/urandom | head -c 10
|
||||
echo
|
||||
)"
|
||||
|
||||
cd "${omni_source_dir}"
|
||||
|
||||
# The XPU base image is ~37GB; the default gzip layer exporter is single-threaded
|
||||
# and takes ~16min to compress it. zstd is multi-threaded (uses all cores) at a
|
||||
# similar ratio; set EXPORT_COMPRESSION=uncompressed to skip compression entirely
|
||||
# for local-only images. This requires the containerd image store (buildx docker
|
||||
# driver), which is the default here.
|
||||
EXPORT_COMPRESSION="${EXPORT_COMPRESSION:-zstd}"
|
||||
if [ "${EXPORT_COMPRESSION}" = "uncompressed" ]; then
|
||||
export_args=(--output "type=image,name={{IMAGE}},compression=uncompressed")
|
||||
else
|
||||
export_args=(--output "type=image,name={{IMAGE}},compression=${EXPORT_COMPRESSION},compression-level=3,force-compression=true")
|
||||
fi
|
||||
|
||||
docker_build() {
|
||||
# $1 = image name; remaining args passed through to docker build.
|
||||
local image="$1"
|
||||
shift
|
||||
local out=("${export_args[@]/'{{IMAGE}}'/${image}}")
|
||||
docker build "${out[@]}" "$@" -f docker/Dockerfile.xpu .
|
||||
}
|
||||
|
||||
build_vllm_base() (
|
||||
local vllm_source_dir
|
||||
vllm_source_dir=$(mktemp -d)
|
||||
trap 'rm -rf "${vllm_source_dir}"' EXIT
|
||||
|
||||
git clone --depth 1 --branch "${VLLM_VERSION}" \
|
||||
https://github.com/vllm-project/vllm "${vllm_source_dir}"
|
||||
|
||||
local out=("${export_args[@]/'{{IMAGE}}'/${base_image_name}}")
|
||||
docker build "${out[@]}" \
|
||||
--target vllm-openai \
|
||||
-f "${vllm_source_dir}/docker/Dockerfile.xpu" \
|
||||
"${vllm_source_dir}"
|
||||
)
|
||||
|
||||
# Resolve the vLLM base image, in priority order:
|
||||
# 1. the published upstream image for VLLM_VERSION, pulled fresh (so moving
|
||||
# tags like nightly are refreshed rather than served stale from disk);
|
||||
# 2. a local copy of that same image, when the registry is unreachable;
|
||||
# 3. VLLM_BASE, if that image is already on disk;
|
||||
# 4. otherwise build VLLM_BASE from the matching vLLM source tag (~30min).
|
||||
# VLLM_BASE names the fallback base and defaults to ${local_base_image}; set it
|
||||
# to reuse or produce a base under a different tag.
|
||||
resolve_vllm_base() {
|
||||
if docker pull "${upstream_base_image}"; then
|
||||
base_image_name="${upstream_base_image}"
|
||||
return
|
||||
fi
|
||||
|
||||
# Registry unreachable, but a previous run may have left the image behind.
|
||||
if [ -n "$(docker images -q "${upstream_base_image}")" ]; then
|
||||
echo "WARNING: could not pull ${upstream_base_image}; using local copy" >&2
|
||||
base_image_name="${upstream_base_image}"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "WARNING: ${upstream_base_image} unavailable remotely and locally" >&2
|
||||
base_image_name="${VLLM_BASE:-${local_base_image}}"
|
||||
if [ -z "$(docker images -q "${base_image_name}")" ]; then
|
||||
echo "WARNING: building ${base_image_name} from vLLM ${VLLM_VERSION} source" >&2
|
||||
build_vllm_base
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_vllm_base
|
||||
|
||||
# Try building the docker image
|
||||
docker_build "${image_name}" --build-arg "VLLM_BASE=${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}"
|
||||
|
||||
# Setup cleanup
|
||||
remove_docker_container() {
|
||||
docker rm -f "${container_name}" || true
|
||||
docker image rm -f "${image_name}" || true
|
||||
docker system prune -f || true
|
||||
}
|
||||
trap remove_docker_container EXIT
|
||||
|
||||
HF_CACHE="${HF_CACHE:-$(realpath ~)/.cache/huggingface}"
|
||||
mkdir -p "${HF_CACHE}"
|
||||
HF_MOUNT="/root/.cache/huggingface"
|
||||
|
||||
time timeout -k 30 30m docker run \
|
||||
--device /dev/dri:/dev/dri \
|
||||
--net=host \
|
||||
--ipc=host \
|
||||
-v /dev/dri/by-path:/dev/dri/by-path \
|
||||
-v "${HF_CACHE}:${HF_MOUNT}" \
|
||||
--security-opt seccomp=unconfined \
|
||||
--entrypoint="" \
|
||||
-e VLLM_LOGGING_LEVEL \
|
||||
-e VLLM_OMNI_LOGGING_LEVEL \
|
||||
-e HF_TOKEN \
|
||||
-e ZE_AFFINITY_MASK \
|
||||
--name "${container_name}" \
|
||||
"${image_name}" \
|
||||
bash -c '
|
||||
set -e
|
||||
echo $ZE_AFFINITY_MASK
|
||||
pip install tblib==3.1.0
|
||||
cd /workspace/vllm-omni
|
||||
XPU_TEST_PATHS="tests/diffusion tests/dfx tests/e2e"
|
||||
pytest -v -s $XPU_TEST_PATHS -m "core_model and xpu and B60"
|
||||
pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py
|
||||
pytest -v -s $XPU_TEST_PATHS -m "advanced_model and xpu and B60"
|
||||
pytest -v -s $XPU_TEST_PATHS -m "omni and xpu and B60"
|
||||
'
|
||||
@@ -1,181 +0,0 @@
|
||||
# Uploaded by upload_pipeline.py --upload (not the Buildkite hook).
|
||||
# ``if`` is injected by step ``key`` from skip-ci at upload time.
|
||||
steps:
|
||||
- label: ":buildkit: Build and Push NPU Test Image - A2"
|
||||
key: image-build-a2
|
||||
agents:
|
||||
queue: "ascend-a2b3"
|
||||
resource_class: "npu-1"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
metadata:
|
||||
annotations:
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-config.json: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-run-as-group: "1000"
|
||||
vault.hashicorp.com/agent-run-as-user: "1000"
|
||||
vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
|
||||
vault.hashicorp.com/role: ascend-gha-runners
|
||||
vault.hashicorp.com/secret-volume-path: /home/user/.docker/
|
||||
vault.hashicorp.com/tls-skip-verify: "true"
|
||||
podSpecPatch:
|
||||
volumes:
|
||||
- name: token-vol
|
||||
projected:
|
||||
defaultMode: 420
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
audience: api
|
||||
expirationSeconds: 600
|
||||
path: token
|
||||
env:
|
||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
||||
IMAGE_NAME: "vllm-omni-ci-npu"
|
||||
IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
|
||||
BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
|
||||
command: |
|
||||
set -ex
|
||||
|
||||
echo "--- Building and pushing NPU Test Image"
|
||||
echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
echo "buildkitd address: $${BUILDKITD_ADDR}"
|
||||
|
||||
if ! command -v buildctl &> /dev/null; then
|
||||
echo "Installing buildctl..."
|
||||
mkdir -p /tmp/buildkit
|
||||
BUILDKIT_VERSION="v0.29.0"
|
||||
wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
|
||||
tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
|
||||
cp /tmp/buildkit/bin/buildctl /usr/local/bin/
|
||||
fi
|
||||
|
||||
export DOCKER_CONFIG=/home/user/.docker
|
||||
buildctl \
|
||||
--addr="$${BUILDKITD_ADDR}" \
|
||||
--tlscacert=/home/user/.docker/ca.pem \
|
||||
--tlscert=/home/user/.docker/cert.pem \
|
||||
--tlskey=/home/user/.docker/key.pem \
|
||||
build \
|
||||
--frontend dockerfile.v0 \
|
||||
--local context=. \
|
||||
--local dockerfile=./docker \
|
||||
--opt filename=Dockerfile.npu.ci \
|
||||
--secret id=dockerconfig,src=/home/user/.docker/config.json \
|
||||
--output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
|
||||
--progress=plain
|
||||
|
||||
echo "--- Image pushed successfully"
|
||||
echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
|
||||
- label: ":buildkit: Build and Push NPU Test Image - A3"
|
||||
key: image-build-a3
|
||||
agents:
|
||||
queue: "ascend-a2b3"
|
||||
resource_class: "npu-1"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
metadata:
|
||||
annotations:
|
||||
vault.hashicorp.com/agent-init-first: "true"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-config.json: "0400"
|
||||
vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
|
||||
vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
|
||||
vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
|
||||
vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
vault.hashicorp.com/agent-run-as-group: "1000"
|
||||
vault.hashicorp.com/agent-run-as-user: "1000"
|
||||
vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
|
||||
vault.hashicorp.com/role: ascend-gha-runners
|
||||
vault.hashicorp.com/secret-volume-path: /home/user/.docker/
|
||||
vault.hashicorp.com/tls-skip-verify: "true"
|
||||
podSpecPatch:
|
||||
volumes:
|
||||
- name: token-vol
|
||||
projected:
|
||||
defaultMode: 420
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
audience: api
|
||||
expirationSeconds: 600
|
||||
path: token
|
||||
env:
|
||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
|
||||
IMAGE_NAME: "vllm-omni-ci-npu"
|
||||
IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
|
||||
BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
|
||||
command: |
|
||||
set -ex
|
||||
|
||||
echo "--- Building and pushing NPU Test Image"
|
||||
echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
echo "buildkitd address: $${BUILDKITD_ADDR}"
|
||||
|
||||
if ! command -v buildctl &> /dev/null; then
|
||||
echo "Installing buildctl..."
|
||||
mkdir -p /tmp/buildkit
|
||||
BUILDKIT_VERSION="v0.29.0"
|
||||
wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
|
||||
tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
|
||||
cp /tmp/buildkit/bin/buildctl /usr/local/bin/
|
||||
fi
|
||||
|
||||
export DOCKER_CONFIG=/home/user/.docker
|
||||
buildctl \
|
||||
--addr="$${BUILDKITD_ADDR}" \
|
||||
--tlscacert=/home/user/.docker/ca.pem \
|
||||
--tlscert=/home/user/.docker/cert.pem \
|
||||
--tlskey=/home/user/.docker/key.pem \
|
||||
build \
|
||||
--frontend dockerfile.v0 \
|
||||
--local context=. \
|
||||
--local dockerfile=./docker \
|
||||
--opt filename=Dockerfile.npu.ci.a3 \
|
||||
--secret id=dockerconfig,src=/home/user/.docker/config.json \
|
||||
--output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
|
||||
--progress=plain
|
||||
|
||||
echo "--- Image pushed successfully"
|
||||
echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
|
||||
- label: "Upload Ready Pipeline"
|
||||
depends_on:
|
||||
- image-build-a2
|
||||
- image-build-a3
|
||||
key: upload-ready-pipeline
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
command: |
|
||||
python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-ready.yml
|
||||
|
||||
# L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
- label: "Upload Nightly Pipeline"
|
||||
depends_on:
|
||||
- image-build-a2
|
||||
- image-build-a3
|
||||
key: upload-nightly-pipeline
|
||||
commands:
|
||||
- python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -1,110 +0,0 @@
|
||||
# steps:
|
||||
# - label: ":buildkit: Build and Push NPU Test Image - A3"
|
||||
# key: image-build-a3
|
||||
# agents:
|
||||
# queue: "ascend-a3"
|
||||
# resource_class: "npu-2"
|
||||
# plugins:
|
||||
# - kubernetes:
|
||||
# metadata:
|
||||
# annotations:
|
||||
# vault.hashicorp.com/agent-init-first: "true"
|
||||
# vault.hashicorp.com/agent-inject: "true"
|
||||
# vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
|
||||
# vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
|
||||
# vault.hashicorp.com/agent-inject-perms-config.json: "0400"
|
||||
# vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
|
||||
# vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
|
||||
# vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
|
||||
# vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
|
||||
# vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
|
||||
# vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
|
||||
# vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
|
||||
# vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
|
||||
# vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
|
||||
# vault.hashicorp.com/agent-pre-populate-only: "true"
|
||||
# vault.hashicorp.com/agent-run-as-group: "1000"
|
||||
# vault.hashicorp.com/agent-run-as-user: "1000"
|
||||
# vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
|
||||
# vault.hashicorp.com/role: ascend-gha-runners
|
||||
# vault.hashicorp.com/secret-volume-path: /home/user/.docker/
|
||||
# vault.hashicorp.com/tls-skip-verify: "true"
|
||||
# podSpecPatch:
|
||||
# volumes:
|
||||
# - name: token-vol
|
||||
# projected:
|
||||
# defaultMode: 420
|
||||
# sources:
|
||||
# - serviceAccountToken:
|
||||
# audience: api
|
||||
# expirationSeconds: 600
|
||||
# path: token
|
||||
# env:
|
||||
# VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
||||
# IMAGE_NAME: "vllm-omni-ci-npu"
|
||||
# IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
|
||||
# BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
|
||||
# command: |
|
||||
# set -ex
|
||||
|
||||
# echo "--- Building and pushing NPU Test Image"
|
||||
# echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
# echo "buildkitd address: $${BUILDKITD_ADDR}"
|
||||
|
||||
# git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/"
|
||||
|
||||
# if ! command -v buildctl &> /dev/null; then
|
||||
# echo "Installing buildctl..."
|
||||
# mkdir -p /tmp/buildkit
|
||||
# BUILDKIT_VERSION="v0.29.0"
|
||||
# wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
|
||||
# tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
|
||||
# cp /tmp/buildkit/bin/buildctl /usr/local/bin/
|
||||
# fi
|
||||
|
||||
# export DOCKER_CONFIG=/home/user/.docker
|
||||
# buildctl \
|
||||
# --addr="$${BUILDKITD_ADDR}" \
|
||||
# --tlscacert=/home/user/.docker/ca.pem \
|
||||
# --tlscert=/home/user/.docker/cert.pem \
|
||||
# --tlskey=/home/user/.docker/key.pem \
|
||||
# build \
|
||||
# --frontend dockerfile.v0 \
|
||||
# --local context=. \
|
||||
# --local dockerfile=./docker \
|
||||
# --opt filename=Dockerfile.npu.ci.a3 \
|
||||
# --secret id=dockerconfig,src=/home/user/.docker/config.json \
|
||||
# --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
|
||||
# --progress=plain
|
||||
|
||||
# echo "--- Image pushed successfully"
|
||||
# echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
|
||||
|
||||
# # - label: "Upload Ready Pipeline - A3"
|
||||
# # depends_on: image-build-a3
|
||||
# # key: upload-ready-pipeline-a3
|
||||
# # if: build.branch != "main" && build.pull_request.labels includes "npu-test"
|
||||
# # agents:
|
||||
# # queue: "ascend-a3"
|
||||
# # resource_class: "npu-2"
|
||||
# # command: |
|
||||
# # buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml
|
||||
|
||||
# # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
# - label: "Upload Nightly Pipeline"
|
||||
# depends_on: image-build
|
||||
# key: upload-nightly-pipeline
|
||||
# if: >-
|
||||
# (build.branch == "main" && build.env("NIGHTLY") == "1") ||
|
||||
# (build.branch != "main" && (
|
||||
# build.pull_request.labels includes "nightly-test" ||
|
||||
# build.pull_request.labels includes "omni-test" ||
|
||||
# build.pull_request.labels includes "tts-test" ||
|
||||
# build.pull_request.labels includes "diffusion-x2iat-test" ||
|
||||
# build.pull_request.labels includes "diffusion-x2v-test"
|
||||
# ))
|
||||
# commands:
|
||||
# - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
|
||||
# agents:
|
||||
# queue: "ascend-a3"
|
||||
# resource_class: "npu-2"
|
||||
@@ -1,9 +0,0 @@
|
||||
# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml
|
||||
# and injects ``if`` by step ``key`` from skip-ci.
|
||||
steps:
|
||||
- label: ":github: Resolve skip-ci & upload pipeline"
|
||||
key: upload-ci-pipeline
|
||||
commands:
|
||||
- "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/bootstrap-upload-steps.yml"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -1,261 +0,0 @@
|
||||
steps:
|
||||
- group: ":card_index_dividers: Omni Model Test"
|
||||
key: nightly-omni-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "omni-test"
|
||||
steps:
|
||||
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
||||
key: nightly-omni-performance-no-async-chunk
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · Perf Test · Async Chunk"
|
||||
key: nightly-omni-performance-async-chunk
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- >-
|
||||
pytest -s -v
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
-m "npu and A3 and omni"
|
||||
--run-level full_model
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
# Pin the Seed-TTS scorer to the second card: _get_eval_device() otherwise
|
||||
# auto-picks npu:0, which the omni server under test already fills with its
|
||||
# KV cache, so Whisper OOMs on `.to(device)` and every clip reports
|
||||
# "ASR / WER failed".
|
||||
- export SEED_TTS_WER_EVAL=1
|
||||
- export SEED_TTS_EVAL_DEVICE=npu:1
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5-npu
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
|
||||
key: nightly-omni-performance-minicpmo-4-5-npu-a2
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a2b3_npu_1
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
HF_HUB_OFFLINE: "1"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A2" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: nightly-tts-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
steps:
|
||||
- label: ":full_moon: TTS · Perf Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- group: ":card_index_dividers: Diffusion I2I Model Test"
|
||||
key: nightly-HunyuanImage3-x2i-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2iat-test"
|
||||
steps:
|
||||
- label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
|
||||
HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
|
||||
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
|
||||
- group: ":card_index_dividers: Diffusion Benchmark Perf Test"
|
||||
key: nightly-diffusion-benchmark-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2iat-test"
|
||||
steps:
|
||||
- label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
- group: ":card_index_dividers: Diffusion X2V Model Test"
|
||||
key: nightly-diffusion-x2v-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2v-test"
|
||||
steps:
|
||||
- label: ":full_moon: Diffusion X2V · Function Test"
|
||||
mirror_hardwares: a2b3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test"
|
||||
key: nightly-diffusion-x2v-performance
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
@@ -1,64 +0,0 @@
|
||||
steps:
|
||||
- label: "Platforms Test"
|
||||
key: ready-platforms-test
|
||||
depends_on: upload-ready-pipeline
|
||||
mirror_hardwares: a3_npu_2
|
||||
commands:
|
||||
- >-
|
||||
pytest -s -v
|
||||
tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
|
||||
-m 'core_model and npu and A3 and omni'
|
||||
--run-level 'core_model'
|
||||
|
||||
- label: "Diffusion NPU layer tests"
|
||||
key: ready-diffusion-npu-layer-tests
|
||||
depends_on: upload-ready-pipeline
|
||||
mirror_hardwares: a3_npu_2
|
||||
commands:
|
||||
- >-
|
||||
pytest -s -v
|
||||
tests/diffusion/layers/test_activation.py
|
||||
tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
|
||||
-m 'core_model and npu and A3'
|
||||
--run-level 'core_model'
|
||||
|
||||
- group: ":card_index_dividers: Omni Model Test"
|
||||
key: ready-omni-group
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||
- label: "Omni · MiniCPM-o 4.5 Perf Test"
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'
|
||||
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: ready-tts-group
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model'
|
||||
@@ -0,0 +1,236 @@
|
||||
steps:
|
||||
- label: ":docker: Build image"
|
||||
key: image-build
|
||||
commands:
|
||||
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
|
||||
- "docker build --file docker/Dockerfile.ci -t vllm-omni-ci ."
|
||||
- "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
|
||||
- "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# - label: "Test on NPU"
|
||||
# depends_on: ~
|
||||
# key: npu-test
|
||||
# commands:
|
||||
# - ".buildkite/scripts/hardware_ci/run_npu_test.sh"
|
||||
# agents:
|
||||
# queue: "ascend"
|
||||
|
||||
- label: "Simple Unit Test"
|
||||
depends_on: ~
|
||||
commands:
|
||||
- ".buildkite/scripts/simple_test.sh"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
- label: "Diffusion Model Test"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion Model CPU offloading Test"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Audio Generation Model Test"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_stable_audio_model.py
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion Cache Backend Test"
|
||||
timeout_in_minutes: 15
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_cache_dit.py tests/e2e/offline_inference/test_teacache.py
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion Sequence Parallelism Test"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
|
||||
agents:
|
||||
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion Tensor Parallelism Test"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_zimage_tensor_parallel.py
|
||||
agents:
|
||||
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion GPU Worker Test"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- pytest -s -v tests/diffusion/test_gpu_diffusion_worker.py
|
||||
agents:
|
||||
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Omni Model Test"
|
||||
timeout_in_minutes: 15
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
|
||||
agents:
|
||||
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Omni Model Test with H100"
|
||||
timeout_in_minutes: 30
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_async_omni.py
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion Image Edit Test with H100 (1 GPU)"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/online_serving/test_i2i_multi_image_input.py
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
@@ -1,132 +0,0 @@
|
||||
steps:
|
||||
- input: "Provide Release version here"
|
||||
id: input-release-version
|
||||
fields:
|
||||
- text: "What is the release version?"
|
||||
key: release-version
|
||||
|
||||
- label: "Build and upload wheel"
|
||||
depends_on: ~
|
||||
key: "build-wheel"
|
||||
agents:
|
||||
queue: cpu_queue_release
|
||||
commands:
|
||||
- "curl -LsSf https://astral.sh/uv/install.sh | sh"
|
||||
- 'export PATH="$HOME/.local/bin:$PATH"'
|
||||
- "uv venv --python=3.12 && source .venv/bin/activate"
|
||||
- "uv pip install --upgrade build"
|
||||
- "python3 -m build"
|
||||
- "bash .buildkite/release/scripts/upload-nightly-wheels.sh"
|
||||
|
||||
- label: "Generate and upload wheel indices"
|
||||
depends_on: "build-wheel"
|
||||
allow_dependency_failure: true
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/release/scripts/generate-and-upload-nightly-index.sh"
|
||||
|
||||
- block: "Upload release wheels to PyPI"
|
||||
depends_on: "build-wheel"
|
||||
key: block-upload-release-wheels
|
||||
|
||||
- label: "Upload release wheels to PyPI"
|
||||
depends_on:
|
||||
- block-upload-release-wheels
|
||||
- input-release-version
|
||||
key: upload-release-wheels
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh"
|
||||
|
||||
# Auto-skip when NIGHTLY=1 (scheduled); otherwise require manual unblock.
|
||||
- block: "Unblock to build release Docker images"
|
||||
depends_on: ~
|
||||
key: block-build-release-images
|
||||
if: build.env("NIGHTLY") != "1"
|
||||
|
||||
- group: "Build release Docker images"
|
||||
key: "build-release-images"
|
||||
depends_on: block-build-release-images
|
||||
allow_dependency_failure: true
|
||||
steps:
|
||||
- label: "Build release image - x86_64"
|
||||
key: build-release-image-x86
|
||||
depends_on: ~
|
||||
agents:
|
||||
queue: cpu_queue_release
|
||||
commands:
|
||||
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
|
||||
- |
|
||||
DOCKER_BUILDKIT=1 docker build \
|
||||
--build-arg max_jobs=16 \
|
||||
--tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
|
||||
--progress plain \
|
||||
-f docker/Dockerfile.cuda .
|
||||
- "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
|
||||
# Fallback default tag (no arch suffix) if arm64 build fails.
|
||||
- "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
|
||||
- "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
|
||||
- label: "Build release image - aarch64"
|
||||
key: build-release-image-arm64
|
||||
depends_on: ~
|
||||
agents:
|
||||
queue: arm64_cpu_queue_release
|
||||
commands:
|
||||
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
|
||||
- |
|
||||
DOCKER_BUILDKIT=1 docker build \
|
||||
--build-arg max_jobs=16 \
|
||||
--tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
|
||||
--progress plain \
|
||||
-f docker/Dockerfile.cuda .
|
||||
- "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
|
||||
# NIGHTLY=1 only: promote ECR images to DockerHub nightly tags, then prune.
|
||||
- label: "Publish nightly multi-arch image to DockerHub"
|
||||
depends_on:
|
||||
- build-release-image-x86
|
||||
- build-release-image-arm64
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
key: publish-nightly-images-dockerhub
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/release/scripts/publish-release-images.sh nightly"
|
||||
- "bash .buildkite/release/scripts/cleanup-nightly-builds.sh"
|
||||
plugins:
|
||||
- docker-login#v3.0.0:
|
||||
username: vllmbot
|
||||
password-env: DOCKERHUB_TOKEN
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
DOCKERHUB_USERNAME: "vllmbot"
|
||||
|
||||
- block: "Publish release images to DockerHub"
|
||||
key: block-publish-release-images
|
||||
depends_on:
|
||||
- build-release-image-x86
|
||||
- build-release-image-arm64
|
||||
- input-release-version
|
||||
|
||||
- label: "Publish release images to DockerHub"
|
||||
depends_on:
|
||||
- block-publish-release-images
|
||||
key: publish-release-images-dockerhub
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/release/scripts/publish-release-images.sh"
|
||||
plugins:
|
||||
- docker-login#v3.0.0:
|
||||
username: vllmbot
|
||||
password-env: DOCKERHUB_TOKEN
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
DOCKERHUB_USERNAME: "vllmbot"
|
||||
@@ -1,147 +0,0 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Clean up old nightly builds from DockerHub (vllm/vllm-omni), keeping the
|
||||
# newest 14 tags with the nightly- prefix.
|
||||
|
||||
set -ex
|
||||
|
||||
TAG_PREFIX="nightly-"
|
||||
REPO="vllm/vllm-omni"
|
||||
REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags"
|
||||
|
||||
echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
|
||||
|
||||
if [ -z "$DOCKERHUB_TOKEN" ]; then
|
||||
echo "Error: DOCKERHUB_TOKEN environment variable is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$DOCKERHUB_USERNAME" ]; then
|
||||
echo "Error: DOCKERHUB_USERNAME environment variable is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Getting DockerHub bearer token..."
|
||||
set +x
|
||||
BEARER_TOKEN=$(curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \
|
||||
"https://hub.docker.com/v2/users/login" | jq -r '.token')
|
||||
set -x
|
||||
|
||||
if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then
|
||||
echo "Error: Failed to get DockerHub bearer token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
get_all_tags() {
|
||||
local page=1
|
||||
local all_tags=""
|
||||
# Hub `name` is a partial match (not a strict prefix). Use it to narrow the
|
||||
# list, then enforce startswith() client-side. Exhaust API pages; do not
|
||||
# stop merely because a page has no matching tags after filtering.
|
||||
local name_query
|
||||
name_query=$(printf '%s' "$TAG_PREFIX" | jq -sRr @uri)
|
||||
|
||||
while true; do
|
||||
set +x
|
||||
local response
|
||||
response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \
|
||||
"$REPO_API_URL?page=$page&page_size=100&name=${name_query}")
|
||||
set -x
|
||||
|
||||
local page_count
|
||||
page_count=$(echo "$response" | jq -r '.results | length')
|
||||
if [ -z "$page_count" ] || [ "$page_count" = "0" ] || [ "$page_count" = "null" ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
local tags
|
||||
tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" \
|
||||
'.results[] | select(.name | startswith($prefix)) | "\(.last_updated)|\(.name)"')
|
||||
|
||||
if [ -n "$tags" ]; then
|
||||
all_tags="$all_tags$tags"$'\n'
|
||||
fi
|
||||
page=$((page + 1))
|
||||
done
|
||||
|
||||
echo "$all_tags" | sort -r | cut -d'|' -f2
|
||||
}
|
||||
|
||||
delete_tag() {
|
||||
local tag_name="$1"
|
||||
echo "Deleting tag: $tag_name"
|
||||
|
||||
local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name"
|
||||
set +x
|
||||
local response http_code body
|
||||
response=$(curl -s -w "\n%{http_code}" -X DELETE \
|
||||
-H "Authorization: Bearer $BEARER_TOKEN" "$delete_url")
|
||||
set -x
|
||||
http_code=$(printf '%s\n' "$response" | tail -n1)
|
||||
body=$(printf '%s\n' "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
|
||||
echo "Successfully deleted tag: $tag_name (HTTP ${http_code})"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local detail=""
|
||||
if echo "$body" | jq -e '.detail' > /dev/null 2>&1; then
|
||||
detail=$(echo "$body" | jq -r '.detail')
|
||||
elif [ -n "$body" ]; then
|
||||
detail="$body"
|
||||
else
|
||||
detail="(empty response body)"
|
||||
fi
|
||||
echo "Error: Failed to delete tag $tag_name (HTTP ${http_code}): ${detail}"
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "Fetching all tags from DockerHub..."
|
||||
all_tags=$(get_all_tags)
|
||||
|
||||
if [ -z "$all_tags" ]; then
|
||||
echo "No tags found to clean up"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
total_tags=$(echo "$all_tags" | wc -l)
|
||||
echo "Found $total_tags tags"
|
||||
|
||||
tags_to_keep=14
|
||||
tags_to_delete=$((total_tags - tags_to_keep))
|
||||
|
||||
if [ $tags_to_delete -le 0 ]; then
|
||||
echo "No tags need to be deleted (only $total_tags tags found, keeping $tags_to_keep)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Will delete $tags_to_delete old tags, keeping the newest $tags_to_keep"
|
||||
|
||||
tags_to_delete_list=$(echo "$all_tags" | tail -n +$((tags_to_keep + 1)))
|
||||
|
||||
if [ -z "$tags_to_delete_list" ]; then
|
||||
echo "No tags to delete"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Deleting old tags..."
|
||||
delete_failures=0
|
||||
while IFS= read -r tag; do
|
||||
if [ -n "$tag" ]; then
|
||||
if ! delete_tag "$tag"; then
|
||||
delete_failures=$((delete_failures + 1))
|
||||
fi
|
||||
sleep 1
|
||||
fi
|
||||
done <<< "$tags_to_delete_list"
|
||||
|
||||
if [ "$delete_failures" -gt 0 ]; then
|
||||
echo "Cleanup finished with ${delete_failures} deletion failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Cleanup completed successfully"
|
||||
@@ -1,88 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -ex
|
||||
|
||||
# Generate and upload wheel indices for all vllm-omni wheels in the commit directory.
|
||||
# This script should run once after all wheels have been built and uploaded.
|
||||
# All paths are under the omni/ prefix in the vllm-wheels S3 bucket.
|
||||
|
||||
# ======== setup ========
|
||||
|
||||
BUCKET="vllm-wheels"
|
||||
INDICES_OUTPUT_DIR="indices"
|
||||
PYTHON="${PYTHON_PROG:-python3}"
|
||||
SUBPATH="omni/$BUILDKITE_COMMIT"
|
||||
S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
|
||||
|
||||
# detect if python3.12+ is available
|
||||
has_new_python=$($PYTHON -c "print(1 if __import__('sys').version_info >= (3,12) else 0)")
|
||||
if [[ "$has_new_python" -eq 0 ]]; then
|
||||
# use new python from docker
|
||||
docker pull python:3-slim
|
||||
PYTHON="docker run --rm --user $(id -u):$(id -g) -v $(pwd):/app -w /app python:3-slim python3"
|
||||
fi
|
||||
|
||||
echo "Using python interpreter: $PYTHON"
|
||||
echo "Python version: $($PYTHON --version)"
|
||||
|
||||
# ======== generate and upload indices ========
|
||||
|
||||
# list all wheels in the commit directory
|
||||
echo "Existing wheels on S3:"
|
||||
aws s3 ls "$S3_COMMIT_PREFIX" || echo "(no objects found)"
|
||||
obj_json="objects.json"
|
||||
aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$SUBPATH/" --delimiter / --output json > "$obj_json"
|
||||
mkdir -p "$INDICES_OUTPUT_DIR"
|
||||
|
||||
# HACK: we do not need regex module here, but it is required by pre-commit hook
|
||||
# To avoid any external dependency, we simply replace it back to the stdlib re module
|
||||
sed -i.bak 's/import regex as re/import re/g' .buildkite/release/scripts/generate-nightly-index.py && rm -f .buildkite/release/scripts/generate-nightly-index.py.bak
|
||||
|
||||
# Generate indices -- the version is just the commit hash (not omni/{commit})
|
||||
# because relative paths are computed between the index and wheel directories,
|
||||
# both of which live under the omni/ prefix in S3.
|
||||
$PYTHON .buildkite/release/scripts/generate-nightly-index.py \
|
||||
--version "$BUILDKITE_COMMIT" \
|
||||
--current-objects "$obj_json" \
|
||||
--output-dir "$INDICES_OUTPUT_DIR" \
|
||||
--comment "commit $BUILDKITE_COMMIT"
|
||||
|
||||
# copy indices to /omni/{commit}/ unconditionally
|
||||
echo "Uploading indices to $S3_COMMIT_PREFIX"
|
||||
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
|
||||
|
||||
# copy to /omni/nightly/ when NIGHTLY=1
|
||||
if [[ "${NIGHTLY:-}" == "1" ]]; then
|
||||
echo "Uploading indices to overwrite /omni/nightly/"
|
||||
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/omni/nightly/"
|
||||
fi
|
||||
|
||||
# detect version from any wheel in the commit directory
|
||||
first_wheel_key=$($PYTHON -c "import json; obj=json.load(open('$obj_json')); print(next((c['Key'] for c in obj.get('Contents', []) if c['Key'].endswith('.whl')), ''))")
|
||||
if [[ -z "$first_wheel_key" ]]; then
|
||||
echo "Error: No wheels found in $S3_COMMIT_PREFIX"
|
||||
exit 1
|
||||
fi
|
||||
first_wheel=$(basename "$first_wheel_key")
|
||||
aws s3 cp "s3://$BUCKET/${first_wheel_key}" "/tmp/${first_wheel}"
|
||||
version=$(unzip -p "/tmp/${first_wheel}" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2)
|
||||
rm -f "/tmp/${first_wheel}"
|
||||
echo "Version in wheel: $version"
|
||||
pure_version="${version%%+*}"
|
||||
echo "Pure version (without variant): $pure_version"
|
||||
|
||||
# re-generate and copy to /omni/{version}/ only if it does not have "dev" in the version
|
||||
if [[ "$version" != *"dev"* ]]; then
|
||||
s3_version="v$pure_version"
|
||||
echo "Re-generating indices for /omni/$s3_version/"
|
||||
rm -rf "${INDICES_OUTPUT_DIR:?}"
|
||||
mkdir -p "$INDICES_OUTPUT_DIR"
|
||||
# wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path
|
||||
$PYTHON .buildkite/release/scripts/generate-nightly-index.py \
|
||||
--version "$s3_version" \
|
||||
--wheel-dir "$BUILDKITE_COMMIT" \
|
||||
--current-objects "$obj_json" \
|
||||
--output-dir "$INDICES_OUTPUT_DIR" \
|
||||
--comment "version $pure_version"
|
||||
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/omni/$s3_version/"
|
||||
fi
|
||||
@@ -1,192 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def normalize_package_name(name: str) -> str:
|
||||
"""Normalize package name per PEP 503."""
|
||||
return re.sub(r"[-_.]+", "-", name).lower()
|
||||
|
||||
|
||||
if not sys.version_info >= (3, 12):
|
||||
raise RuntimeError("This script requires Python 3.12 or higher.")
|
||||
|
||||
INDEX_HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html>
|
||||
<!-- {comment} -->
|
||||
<meta name="pypi:repository-version" content="1.0">
|
||||
<body>
|
||||
{items}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class WheelFileInfo:
|
||||
package_name: str
|
||||
version: str
|
||||
build_tag: str | None
|
||||
python_tag: str
|
||||
abi_tag: str
|
||||
platform_tag: str
|
||||
filename: str
|
||||
|
||||
|
||||
def parse_from_filename(file: str) -> WheelFileInfo:
|
||||
"""
|
||||
Parse wheel filename per PEP 427:
|
||||
{package_name}-{version}(-{build_tag})?-{python_tag}-{abi_tag}-{platform_tag}.whl
|
||||
"""
|
||||
wheel_file_re = re.compile(
|
||||
r"^(?P<package_name>.+)-(?P<version>[^-]+?)(-(?P<build_tag>[^-]+))?-(?P<python_tag>[^-]+)-(?P<abi_tag>[^-]+)-(?P<platform_tag>[^-]+)\.whl$"
|
||||
)
|
||||
match = wheel_file_re.match(file)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid wheel file name: {file}")
|
||||
|
||||
return WheelFileInfo(
|
||||
package_name=match.group("package_name"),
|
||||
version=match.group("version"),
|
||||
build_tag=match.group("build_tag"),
|
||||
python_tag=match.group("python_tag"),
|
||||
abi_tag=match.group("abi_tag"),
|
||||
platform_tag=match.group("platform_tag"),
|
||||
filename=file,
|
||||
)
|
||||
|
||||
|
||||
def generate_project_list(package_names: list[str], comment: str = "") -> str:
|
||||
"""Generate top-level PEP 503 project list HTML."""
|
||||
href_tags = []
|
||||
for name in sorted(package_names):
|
||||
href_tags.append(f' <a href="{name}/">{name}/</a><br/>')
|
||||
return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
|
||||
|
||||
|
||||
def generate_package_index(
|
||||
wheel_files: list[WheelFileInfo],
|
||||
wheel_base_dir: Path,
|
||||
index_base_dir: Path,
|
||||
comment: str = "",
|
||||
) -> tuple[str, str]:
|
||||
"""Generate package index HTML and metadata JSON linking to wheel files."""
|
||||
href_tags = []
|
||||
metadata = []
|
||||
for file in sorted(wheel_files, key=lambda x: x.filename):
|
||||
relative_path = wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename
|
||||
# handle '+' in URL; avoid double-encoding '/' and '%2B' (AWS S3 behavior)
|
||||
file_path_quoted = quote(relative_path.as_posix(), safe=":%/")
|
||||
href_tags.append(f' <a href="{file_path_quoted}">{file.filename}</a><br/>')
|
||||
file_meta = asdict(file)
|
||||
file_meta["path"] = file_path_quoted
|
||||
metadata.append(file_meta)
|
||||
index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
|
||||
metadata_str = json.dumps(metadata, indent=2)
|
||||
return index_str, metadata_str
|
||||
|
||||
|
||||
def generate_index(
|
||||
whl_files: list[str],
|
||||
wheel_base_dir: Path,
|
||||
index_base_dir: Path,
|
||||
comment: str = "",
|
||||
):
|
||||
"""
|
||||
Generate PEP 503 index for all wheel files.
|
||||
|
||||
Output structure:
|
||||
index_base_dir/
|
||||
index.html # project list linking to vllm-omni/
|
||||
vllm-omni/
|
||||
index.html # package index linking to wheel files
|
||||
metadata.json # machine-readable metadata
|
||||
"""
|
||||
parsed_files = [parse_from_filename(f) for f in whl_files]
|
||||
|
||||
if not parsed_files:
|
||||
print("No wheel files found, skipping index generation.")
|
||||
return
|
||||
|
||||
comment_str = f" ({comment})" if comment else ""
|
||||
comment_tmpl = f"Generated on {datetime.now().isoformat()}{comment_str}"
|
||||
|
||||
# Group by normalized package name
|
||||
packages: dict[str, list[WheelFileInfo]] = {}
|
||||
for file in parsed_files:
|
||||
name = normalize_package_name(file.package_name)
|
||||
packages.setdefault(name, []).append(file)
|
||||
|
||||
print(f"Found packages: {list(packages.keys())}")
|
||||
|
||||
# Generate per-package index
|
||||
for package, files in packages.items():
|
||||
package_dir = index_base_dir / package
|
||||
package_dir.mkdir(parents=True, exist_ok=True)
|
||||
index_str, metadata_str = generate_package_index(files, wheel_base_dir, package_dir, comment)
|
||||
with open(package_dir / "index.html", "w") as f:
|
||||
f.write(index_str)
|
||||
with open(package_dir / "metadata.json", "w") as f:
|
||||
f.write(metadata_str)
|
||||
|
||||
# Generate top-level project list
|
||||
project_list_str = generate_project_list(sorted(packages.keys()), comment_tmpl)
|
||||
with open(index_base_dir / "index.html", "w") as f:
|
||||
f.write(project_list_str)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Generate PEP 503 wheel index from S3 object listing.")
|
||||
parser.add_argument("--version", type=str, required=True, help="Version string (e.g., commit hash)")
|
||||
parser.add_argument("--current-objects", type=str, required=True, help="Path to JSON from S3 list-objects-v2")
|
||||
parser.add_argument("--output-dir", type=str, required=True, help="Directory to write index files")
|
||||
parser.add_argument("--wheel-dir", type=str, default=None, help="Wheel directory (defaults to --version)")
|
||||
parser.add_argument("--comment", type=str, default="", help="Comment for generated HTML")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
version = args.version
|
||||
if "\\" in version or "/" in version:
|
||||
raise ValueError("Version string must not contain slashes or backslashes.")
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(args.current_objects) as f:
|
||||
current_objects: dict[str, list[dict[str, Any]]] = json.load(f)
|
||||
|
||||
wheel_files = [
|
||||
item["Key"].split("/")[-1] for item in current_objects.get("Contents", []) if item["Key"].endswith(".whl")
|
||||
]
|
||||
|
||||
print(f"Found {len(wheel_files)} wheel files for version {version}: {wheel_files}")
|
||||
|
||||
# For release versions, filter to only matching non-dev wheels
|
||||
PY_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+([a-zA-Z0-9.+-]*)?$")
|
||||
if PY_VERSION_RE.match(version):
|
||||
wheel_files = [f for f in wheel_files if version in f and "dev" not in f]
|
||||
print(f"Non-nightly version detected, wheel files used: {wheel_files}")
|
||||
else:
|
||||
print("Nightly version detected, keeping all wheel files.")
|
||||
|
||||
wheel_dir = (args.wheel_dir or version).strip().rstrip("/")
|
||||
wheel_base_dir = Path(output_dir).parent / wheel_dir
|
||||
index_base_dir = Path(output_dir)
|
||||
|
||||
generate_index(
|
||||
whl_files=wheel_files,
|
||||
wheel_base_dir=wheel_base_dir,
|
||||
index_base_dir=index_base_dir,
|
||||
comment=args.comment.strip(),
|
||||
)
|
||||
print(f"Successfully generated index in {output_dir}")
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Publish Docker images from ECR staging to DockerHub.
|
||||
#
|
||||
# Usage:
|
||||
# publish-release-images.sh # release mode (default)
|
||||
# publish-release-images.sh release
|
||||
# publish-release-images.sh nightly
|
||||
#
|
||||
# Release tags: latest, v<version>
|
||||
# Nightly tags: nightly, nightly-<commit>
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODE="${1:-release}"
|
||||
|
||||
DOCKERHUB_REPO="vllm/vllm-omni"
|
||||
ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
|
||||
COMMIT="$BUILDKITE_COMMIT"
|
||||
PRIMARY_TAGS=()
|
||||
EXTRA_MANIFEST_TAGS=()
|
||||
|
||||
case "${MODE}" in
|
||||
release)
|
||||
RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
|
||||
if [ -z "${RELEASE_VERSION}" ]; then
|
||||
echo "ERROR: release-version metadata not set"
|
||||
exit 1
|
||||
fi
|
||||
PRIMARY_TAGS=("latest" "v${RELEASE_VERSION}")
|
||||
;;
|
||||
nightly)
|
||||
# Commit-pinned manifest reuses nightly-{x86_64,aarch64} arch images.
|
||||
PRIMARY_TAGS=("nightly")
|
||||
EXTRA_MANIFEST_TAGS=("nightly-${COMMIT}")
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {release|nightly}"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "========================================"
|
||||
echo "Publishing ${MODE} images to ${DOCKERHUB_REPO}"
|
||||
echo " Commit: ${COMMIT}"
|
||||
echo " Primary tags: ${PRIMARY_TAGS[*]}"
|
||||
if [ ${#EXTRA_MANIFEST_TAGS[@]} -gt 0 ]; then
|
||||
echo " Extra manifests: ${EXTRA_MANIFEST_TAGS[*]}"
|
||||
fi
|
||||
echo "========================================"
|
||||
|
||||
aws ecr-public get-login-password --region us-east-1 | \
|
||||
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
|
||||
|
||||
SRC_X86="${ECR_REPO}:${COMMIT}-x86_64"
|
||||
SRC_ARM="${ECR_REPO}:${COMMIT}-aarch64"
|
||||
|
||||
docker pull "${SRC_X86}"
|
||||
docker pull "${SRC_ARM}"
|
||||
|
||||
publish_manifest() {
|
||||
local tag="$1"
|
||||
local arch_tag_base="$2"
|
||||
docker manifest rm "${DOCKERHUB_REPO}:${tag}" || true
|
||||
docker manifest create \
|
||||
"${DOCKERHUB_REPO}:${tag}" \
|
||||
"${DOCKERHUB_REPO}:${arch_tag_base}-x86_64" \
|
||||
"${DOCKERHUB_REPO}:${arch_tag_base}-aarch64"
|
||||
docker manifest push "${DOCKERHUB_REPO}:${tag}"
|
||||
}
|
||||
|
||||
for tag in "${PRIMARY_TAGS[@]}"; do
|
||||
docker tag "${SRC_X86}" "${DOCKERHUB_REPO}:${tag}-x86_64"
|
||||
docker tag "${SRC_ARM}" "${DOCKERHUB_REPO}:${tag}-aarch64"
|
||||
docker push "${DOCKERHUB_REPO}:${tag}-x86_64"
|
||||
docker push "${DOCKERHUB_REPO}:${tag}-aarch64"
|
||||
publish_manifest "${tag}" "${tag}"
|
||||
done
|
||||
|
||||
ARCH_BASE="${PRIMARY_TAGS[0]}"
|
||||
for tag in "${EXTRA_MANIFEST_TAGS[@]}"; do
|
||||
publish_manifest "${tag}" "${ARCH_BASE}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Successfully published ${MODE} images"
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
set -e
|
||||
|
||||
BUCKET="vllm-wheels"
|
||||
SUBPATH="omni/$BUILDKITE_COMMIT"
|
||||
S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
|
||||
|
||||
RELEASE_VERSION=$(buildkite-agent meta-data get release-version | sed 's/^v//')
|
||||
if [[ -z "$RELEASE_VERSION" ]]; then
|
||||
echo "[FATAL] release-version metadata not set."
|
||||
exit 1
|
||||
fi
|
||||
echo "Release version: $RELEASE_VERSION"
|
||||
|
||||
if [[ -z "$PYPI_TOKEN" ]]; then
|
||||
echo "[FATAL] PYPI_TOKEN is not set."
|
||||
exit 1
|
||||
else
|
||||
export TWINE_USERNAME="__token__"
|
||||
export TWINE_PASSWORD="$PYPI_TOKEN"
|
||||
fi
|
||||
|
||||
set -x
|
||||
|
||||
if ! command -v uv &> /dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
fi
|
||||
|
||||
uv venv --python 3.12 /tmp/vllm-omni-release-env
|
||||
source /tmp/vllm-omni-release-env/bin/activate
|
||||
uv pip install twine
|
||||
|
||||
DIST_DIR=/tmp/vllm-omni-release-dist
|
||||
mkdir -p "$DIST_DIR"
|
||||
|
||||
echo "Downloading wheels from S3:"
|
||||
aws s3 ls "$S3_COMMIT_PREFIX"
|
||||
aws s3 cp --recursive --exclude "*" --include "vllm_omni-${RELEASE_VERSION}*.whl" --exclude "*dev*" "$S3_COMMIT_PREFIX" "$DIST_DIR"
|
||||
ls -la "$DIST_DIR"
|
||||
|
||||
PYPI_WHEEL_FILES=$(find "$DIST_DIR" -name "vllm_omni-${RELEASE_VERSION}*.whl" -not -name "*+*")
|
||||
if [[ -z "$PYPI_WHEEL_FILES" ]]; then
|
||||
echo "[FATAL] No wheels found for version ${RELEASE_VERSION}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 -m twine check $PYPI_WHEEL_FILES
|
||||
python3 -m twine upload --non-interactive --verbose $PYPI_WHEEL_FILES
|
||||
echo "Wheels uploaded to PyPI"
|
||||
@@ -1,33 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -ex
|
||||
|
||||
# Upload a single wheel to S3 under the omni/ prefix.
|
||||
# Index generation is handled separately by generate-and-upload-nightly-index.sh.
|
||||
|
||||
BUCKET="vllm-wheels"
|
||||
SUBPATH="omni/$BUILDKITE_COMMIT"
|
||||
S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
|
||||
|
||||
# ========= collect & upload the wheel ==========
|
||||
|
||||
# python3 -m build outputs to dist/ by default
|
||||
wheel_files=(dist/*.whl)
|
||||
|
||||
# Check that exactly one wheel is found
|
||||
if [[ ${#wheel_files[@]} -ne 1 ]]; then
|
||||
echo "Error: Expected exactly one wheel file in dist/, but found ${#wheel_files[@]}"
|
||||
exit 1
|
||||
fi
|
||||
wheel="${wheel_files[0]}"
|
||||
|
||||
echo "Uploading wheel: $wheel"
|
||||
|
||||
# Extract the version from the wheel
|
||||
version=$(unzip -p "$wheel" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2)
|
||||
echo "Version in wheel: $version"
|
||||
|
||||
# Upload wheel to S3
|
||||
aws s3 cp "$wheel" "$S3_COMMIT_PREFIX"
|
||||
|
||||
echo "Wheel uploaded to $S3_COMMIT_PREFIX. Index generation is handled by a separate step."
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
#!/bin/bash
|
||||
# Helper function to safely login to ECR Public with per-job config isolation
|
||||
# Uses DOCKER_CONFIG environment variable to prevent race conditions
|
||||
#
|
||||
# This script prevents the "device or resource busy" error by giving each
|
||||
# Buildkite job its own isolated Docker config directory.
|
||||
#
|
||||
# Usage:
|
||||
# source docker_login_ecr_public.sh && safe_docker_login_ecr_public
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
ECR_REGISTRY="public.ecr.aws"
|
||||
|
||||
setup_isolated_docker_config() {
|
||||
# Use BUILDKITE_JOB_ID for job-specific isolation
|
||||
# Fallback to PID if running outside Buildkite
|
||||
local job_id="${BUILDKITE_JOB_ID:-$$}"
|
||||
|
||||
# Set Docker config to job-specific directory
|
||||
export DOCKER_CONFIG="/tmp/docker-config-${job_id}"
|
||||
|
||||
# Create directory if it doesn't exist
|
||||
mkdir -p "$DOCKER_CONFIG"
|
||||
|
||||
echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG"
|
||||
}
|
||||
|
||||
check_docker_auth() {
|
||||
# Check if already authenticated to the given registry
|
||||
# Returns 0 if authenticated, 1 if not
|
||||
local registry="$1"
|
||||
|
||||
# Check if credentials exist in the isolated config
|
||||
if [[ -f "$DOCKER_CONFIG/config.json" ]]; then
|
||||
# Check if registry is present in config
|
||||
if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
safe_docker_login_ecr_public() {
|
||||
# Setup isolated config first
|
||||
setup_isolated_docker_config
|
||||
|
||||
local registry="$ECR_REGISTRY"
|
||||
|
||||
# Check if already authenticated (within this job)
|
||||
if check_docker_auth "$registry"; then
|
||||
echo "[docker-login] Already authenticated to $registry in this job"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Perform login to isolated config directory
|
||||
echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..."
|
||||
if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then
|
||||
echo "[docker-login] Login successful (config: $DOCKER_CONFIG)"
|
||||
return 0
|
||||
else
|
||||
local exit_code=$?
|
||||
echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2
|
||||
return $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
# Execute if run as script (not sourced)
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
safe_docker_login_ecr_public
|
||||
fi
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
#!/bin/bash
|
||||
# vllm-omni customized version
|
||||
# Based on: vllm/.buildkite/scripts/hardware_ci/run-amd-test.sh
|
||||
# Last synced: 2025-12-15
|
||||
# Modifications: docker image name for vllm-omni
|
||||
|
||||
# This script runs test inside the corresponding ROCm docker container.
|
||||
set -o pipefail
|
||||
|
||||
# Export Python path
|
||||
export PYTHONPATH=".."
|
||||
|
||||
# Print ROCm version
|
||||
echo "--- Confirming Clean Initial State"
|
||||
while true; do
|
||||
sleep 3
|
||||
if grep -q clean /opt/amdgpu/etc/gpu_state; then
|
||||
echo "GPUs state is \"clean\""
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- ROCm info"
|
||||
rocminfo
|
||||
|
||||
# cleanup older docker images
|
||||
cleanup_docker() {
|
||||
# Get Docker's root directory
|
||||
docker_root=$(docker info -f '{{.DockerRootDir}}')
|
||||
if [ -z "$docker_root" ]; then
|
||||
echo "Failed to determine Docker root directory."
|
||||
exit 1
|
||||
fi
|
||||
echo "Docker root directory: $docker_root"
|
||||
# Check disk usage of the filesystem where Docker's root directory is located
|
||||
disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
|
||||
# Define the threshold
|
||||
threshold=70
|
||||
if [ "$disk_usage" -gt "$threshold" ]; then
|
||||
echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
|
||||
# Remove dangling images (those that are not tagged and not used by any container)
|
||||
docker image prune -f
|
||||
# Remove unused volumes / force the system prune for old images as well.
|
||||
docker volume prune -f && docker system prune --force --filter "until=72h" --all
|
||||
echo "Docker images and volumes cleanup completed."
|
||||
else
|
||||
echo "Disk usage is below $threshold%. No cleanup needed."
|
||||
fi
|
||||
}
|
||||
|
||||
# Call the cleanup docker function
|
||||
cleanup_docker
|
||||
|
||||
echo "--- Resetting GPUs"
|
||||
|
||||
echo "reset" > /opt/amdgpu/etc/gpu_state
|
||||
|
||||
while true; do
|
||||
sleep 3
|
||||
if grep -q clean /opt/amdgpu/etc/gpu_state; then
|
||||
echo "GPUs state is \"clean\""
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- Pulling container"
|
||||
image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni"
|
||||
container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
|
||||
|
||||
# Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
# Use safe docker login helper to prevent race conditions
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh"
|
||||
safe_docker_login_ecr_public
|
||||
# Pull the container from ECR Public Gallery
|
||||
|
||||
docker pull "${image_name}"
|
||||
|
||||
remove_docker_container() {
|
||||
docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
|
||||
}
|
||||
trap remove_docker_container EXIT
|
||||
|
||||
echo "--- Running container"
|
||||
|
||||
HF_CACHE="$(realpath ~)/huggingface"
|
||||
mkdir -p "${HF_CACHE}"
|
||||
HF_MOUNT="/root/.cache/huggingface"
|
||||
|
||||
commands=$@
|
||||
echo "Commands:$commands"
|
||||
|
||||
PARALLEL_JOB_COUNT=8
|
||||
MYPYTHONPATH=".."
|
||||
|
||||
# Test that we're launching on the machine that has
|
||||
# proper access to GPUs
|
||||
render_gid=$(getent group render | cut -d: -f3)
|
||||
if [[ -z "$render_gid" ]]; then
|
||||
echo "Error: 'render' group not found. This is required for GPU access." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
|
||||
if [[ $commands == *"--shard-id="* ]]; then
|
||||
# assign job count as the number of shards used
|
||||
commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g')
|
||||
for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
|
||||
# assign shard-id for each shard
|
||||
commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g')
|
||||
echo "Shard ${GPU} commands:$commands_gpu"
|
||||
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
|
||||
docker run \
|
||||
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
|
||||
--network=host \
|
||||
--shm-size=16gb \
|
||||
--group-add "$render_gid" \
|
||||
--rm \
|
||||
-e HIP_VISIBLE_DEVICES="${GPU}" \
|
||||
-e HF_TOKEN \
|
||||
-e AWS_ACCESS_KEY_ID \
|
||||
-e AWS_SECRET_ACCESS_KEY \
|
||||
-v "${HF_CACHE}:${HF_MOUNT}" \
|
||||
-e "HF_HOME=${HF_MOUNT}" \
|
||||
-e "PYTHONPATH=${MYPYTHONPATH}" \
|
||||
--name "${container_name}_${GPU}" \
|
||||
"${image_name}" \
|
||||
/bin/bash -c "${commands_gpu}" \
|
||||
|& while read -r line; do echo ">>Shard $GPU: $line"; done &
|
||||
PIDS+=($!)
|
||||
done
|
||||
#wait for all processes to finish and collect exit codes
|
||||
for pid in "${PIDS[@]}"; do
|
||||
wait "${pid}"
|
||||
STATUS+=($?)
|
||||
done
|
||||
for st in "${STATUS[@]}"; do
|
||||
if [[ ${st} -ne 0 ]]; then
|
||||
echo "One of the processes failed with $st"
|
||||
exit "${st}"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
|
||||
docker run \
|
||||
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
|
||||
--network=host \
|
||||
--shm-size=16gb \
|
||||
--group-add "$render_gid" \
|
||||
--rm \
|
||||
-e HF_TOKEN \
|
||||
-e AWS_ACCESS_KEY_ID \
|
||||
-e AWS_SECRET_ACCESS_KEY \
|
||||
-v "${HF_CACHE}:${HF_MOUNT}" \
|
||||
-e "HF_HOME=${HF_MOUNT}" \
|
||||
-e "PYTHONPATH=${MYPYTHONPATH}" \
|
||||
--name "${container_name}" \
|
||||
"${image_name}" \
|
||||
/bin/bash -c "${commands}"
|
||||
fi
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script build the Ascend NPU docker image and run the offline inference inside the container.
|
||||
# It serves a sanity check for compilation and basic model usage.
|
||||
set -ex
|
||||
|
||||
# Base ubuntu image with basic ascend development libraries and python installed
|
||||
VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git"
|
||||
BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2"
|
||||
image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
|
||||
# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
|
||||
container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
|
||||
|
||||
# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards
|
||||
agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
|
||||
echo "agent_idx: ${agent_idx}"
|
||||
builder_name="cachebuilder${agent_idx}"
|
||||
builder_cache_dir="/mnt/docker-cache${agent_idx}"
|
||||
mkdir -p ${builder_cache_dir}
|
||||
|
||||
# Try building the docker image
|
||||
cat <<EOF | DOCKER_BUILDKIT=1 docker build \
|
||||
--add-host pypi-cache:${PYPI_CACHE_HOST} \
|
||||
--builder ${builder_name} --cache-from type=local,src=${builder_cache_dir} \
|
||||
--cache-to type=local,dest=${builder_cache_dir},mode=max \
|
||||
--build-arg BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST}" \
|
||||
--build-arg BUILDKITE_PULL_REQUEST_REPO="${BUILDKITE_PULL_REQUEST_REPO}" \
|
||||
--progress=plain --load -t ${image_name} -f - .
|
||||
FROM ${BASE_IMAGE_NAME}
|
||||
|
||||
# Define environments
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN pip config set global.index-url http://pypi-cache:${PYPI_CACHE_PORT}/pypi/simple && \
|
||||
pip config set global.trusted-host pypi-cache && \
|
||||
apt-get update -y && \
|
||||
apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \
|
||||
rm -rf /var/cache/apt/* && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install for pytest to make the docker build cache layer always valid
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install pytest>=6.0 pytest-cov modelscope
|
||||
|
||||
COPY . .
|
||||
|
||||
# Install vllm-omni
|
||||
WORKDIR /workspace
|
||||
ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git
|
||||
ARG VLLM_OMNI_TAG=main
|
||||
ARG BUILDKITE_PULL_REQUEST
|
||||
ARG BUILDKITE_PULL_REQUEST_REPO
|
||||
RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \
|
||||
if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \
|
||||
echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \
|
||||
git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \
|
||||
cd /workspace/vllm-omni && \
|
||||
git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \
|
||||
git checkout pr-\$BUILDKITE_PULL_REQUEST; \
|
||||
else \
|
||||
echo "Not a PR build, using main branch" && \
|
||||
git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \
|
||||
fi
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh && \
|
||||
export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
|
||||
python3 -m pip install -v -e /workspace/vllm-omni/
|
||||
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
WORKDIR /workspace/vllm-omni
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
EOF
|
||||
|
||||
# Setup cleanup
|
||||
remove_docker_container() {
|
||||
docker rm -f "${container_name}" || true;
|
||||
docker image rm -f "${image_name}" || true;
|
||||
docker system prune -f || true;
|
||||
}
|
||||
trap remove_docker_container EXIT
|
||||
|
||||
# Generate corresponding --device args based on BUILDKITE_AGENT_NAME
|
||||
# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
|
||||
# e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
|
||||
# returns --device /dev/davinci0 --device /dev/davinci1
|
||||
parse_and_gen_devices() {
|
||||
local input="$1"
|
||||
local index cards_num
|
||||
if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then
|
||||
index="${BASH_REMATCH[1]}"
|
||||
cards_num="${BASH_REMATCH[2]}"
|
||||
else
|
||||
echo "parse error" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local devices=""
|
||||
local i=0
|
||||
while (( i < cards_num )); do
|
||||
local dev_idx=$(((index - 1)*cards_num + i ))
|
||||
devices="$devices --device /dev/davinci${dev_idx}"
|
||||
((i++))
|
||||
done
|
||||
|
||||
# trim leading space
|
||||
devices="${devices#"${devices%%[![:space:]]*}"}"
|
||||
# Output devices: assigned to the caller variable
|
||||
printf '%s' "$devices"
|
||||
}
|
||||
|
||||
devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
|
||||
|
||||
# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
|
||||
# This test checks whether the OOT platform interface is functioning properly in conjunction with
|
||||
# the hardware plugin vllm-ascend.
|
||||
hf_model_cache_dir=/mnt/hf_cache${agent_idx}
|
||||
ms_model_cache_dir=/mnt/modelscope${agent_idx}
|
||||
mkdir -p ${hf_model_cache_dir}
|
||||
mkdir -p ${ms_model_cache_dir}
|
||||
docker run \
|
||||
--init \
|
||||
${devices} \
|
||||
--device /dev/davinci_manager \
|
||||
--device /dev/devmm_svm \
|
||||
--device /dev/hisi_hdc \
|
||||
-v /usr/local/dcmi:/usr/local/dcmi \
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
|
||||
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
|
||||
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
|
||||
-v /etc/ascend_install.info:/etc/ascend_install.info \
|
||||
-v ${hf_model_cache_dir}:/root/.cache/huggingface \
|
||||
-v ${ms_model_cache_dir}:/root/.cache/modelscope \
|
||||
--network host \
|
||||
--entrypoint="" \
|
||||
--name "${container_name}" \
|
||||
"${image_name}" \
|
||||
bash -c '
|
||||
set -e
|
||||
VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
|
||||
'
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)"
|
||||
UV_BIN="uv"
|
||||
PYTHON_BOOTSTRAP="${PYTHON:-python3}"
|
||||
VENV_DIR="${VENV_DIR:-${REPO_ROOT}/.venv-simple-test}"
|
||||
TARGET_PY_VERSION="3.12"
|
||||
|
||||
if ! command -v "${PYTHON_BOOTSTRAP}" >/dev/null 2>&1; then
|
||||
PYTHON_BOOTSTRAP="python"
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
# Ensure pip-installed scripts (including uv) are on PATH
|
||||
PYTHON_SCRIPTS="$(${PYTHON_BOOTSTRAP} -c 'import sysconfig; print(sysconfig.get_path("scripts"))')"
|
||||
USER_BASE="$(${PYTHON_BOOTSTRAP} -m site --user-base 2>/dev/null || true)"
|
||||
PATH="${PYTHON_SCRIPTS}:${PATH}"
|
||||
if [[ -n "${USER_BASE}" ]]; then
|
||||
PATH="${USER_BASE}/bin:${PATH}"
|
||||
fi
|
||||
|
||||
if ! command -v "${UV_BIN}" >/dev/null 2>&1; then
|
||||
"${PYTHON_BOOTSTRAP}" -m pip install --upgrade pip
|
||||
"${PYTHON_BOOTSTRAP}" -m pip install uv
|
||||
hash -r
|
||||
fi
|
||||
|
||||
TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)"
|
||||
if [[ -z "${TARGET_PY_PATH}" ]]; then
|
||||
"${UV_BIN}" python install "${TARGET_PY_VERSION}"
|
||||
TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)"
|
||||
fi
|
||||
|
||||
if [[ -z "${TARGET_PY_PATH}" ]]; then
|
||||
echo "Failed to locate python ${TARGET_PY_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "${VENV_DIR}" ]]; then
|
||||
"${UV_BIN}" venv --python "${TARGET_PY_PATH}" "${VENV_DIR}"
|
||||
fi
|
||||
|
||||
VENV_PYTHON="${VENV_DIR}/bin/python"
|
||||
[[ -x "${VENV_PYTHON}" ]] || { echo "Python not found in ${VENV_DIR}"; exit 1; }
|
||||
|
||||
"${UV_BIN}" pip install --python "${VENV_PYTHON}" vllm==0.14.0
|
||||
"${UV_BIN}" pip install --python "${VENV_PYTHON}" -e ".[dev]"
|
||||
|
||||
"${VENV_PYTHON}" -m pytest -v -s tests/entrypoints/
|
||||
"${VENV_PYTHON}" -m pytest -v -s tests/diffusion/cache/
|
||||
"${VENV_PYTHON}" -m pytest -v -s tests/model_executor/models/qwen2_5_omni/test_audio_length.py
|
||||
"${VENV_PYTHON}" -m pytest -v -s tests/worker/
|
||||
"${VENV_PYTHON}" -m pytest -v -s tests/distributed/omni_connectors/test_kv_flow.py
|
||||
@@ -0,0 +1,106 @@
|
||||
steps:
|
||||
|
||||
- label: "Diffusion Model Test"
|
||||
timeout_in_minutes: 20
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export MIOPEN_DEBUG_CONV_DIRECT=0
|
||||
- export MIOPEN_DEBUG_CONV_GEMM=0
|
||||
- export VLLM_ROCM_USE_AITER=1
|
||||
- export VLLM_ROCM_USE_AITER_MHA=1
|
||||
- export VLLM_ROCM_USE_AITER_LINEAR=0
|
||||
- export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
- pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
|
||||
|
||||
- label: "Diffusion Cache Backend Test"
|
||||
timeout_in_minutes: 15
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export MIOPEN_DEBUG_CONV_DIRECT=0
|
||||
- export MIOPEN_DEBUG_CONV_GEMM=0
|
||||
- export VLLM_ROCM_USE_AITER=1
|
||||
- export VLLM_ROCM_USE_AITER_MHA=1
|
||||
- export VLLM_ROCM_USE_AITER_LINEAR=0
|
||||
- export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
- pytest -s -v tests/e2e/offline_inference/test_cache_dit.py tests/e2e/offline_inference/test_teacache.py
|
||||
|
||||
- label: "Diffusion Parallelism Test"
|
||||
timeout_in_minutes: 15
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export MIOPEN_DEBUG_CONV_DIRECT=0
|
||||
- export MIOPEN_DEBUG_CONV_GEMM=0
|
||||
- pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
|
||||
|
||||
- label: "Diffusion GPU Worker Test"
|
||||
timeout_in_minutes: 20
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export MIOPEN_DEBUG_CONV_DIRECT=0
|
||||
- export MIOPEN_DEBUG_CONV_GEMM=0
|
||||
- pytest -s -v tests/diffusion/test_gpu_diffusion_worker.py
|
||||
|
||||
- label: "Omni Model Test Qwen2-5-Omni"
|
||||
timeout_in_minutes: 15
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export MIOPEN_DEBUG_CONV_DIRECT=0
|
||||
- export MIOPEN_DEBUG_CONV_GEMM=0
|
||||
- export VLLM_ROCM_USE_AITER=1
|
||||
- export VLLM_ROCM_USE_AITER_MHA=1
|
||||
- export VLLM_ROCM_USE_AITER_LINEAR=0
|
||||
- export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
|
||||
|
||||
- label: "Omni Model Test Qwen3-Omni"
|
||||
timeout_in_minutes: 15
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export MIOPEN_DEBUG_CONV_DIRECT=0
|
||||
- export MIOPEN_DEBUG_CONV_GEMM=0
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py
|
||||
|
||||
- label: "Diffusion Image Edit Test"
|
||||
timeout_in_minutes: 15
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export MIOPEN_DEBUG_CONV_DIRECT=0
|
||||
- export MIOPEN_DEBUG_CONV_GEMM=0
|
||||
- export VLLM_ROCM_USE_AITER=1
|
||||
- export VLLM_ROCM_USE_AITER_MHA=1
|
||||
- export VLLM_ROCM_USE_AITER_LINEAR=0
|
||||
- export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
- pytest -s -v tests/e2e/online_serving/test_i2i_multi_image_input.py
|
||||
@@ -0,0 +1,53 @@
|
||||
{# vllm-omni customized version
|
||||
Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
|
||||
Last synced: 2025-12-15
|
||||
Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests
|
||||
#}
|
||||
{% set docker_image_amd = "public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT-rocm-omni" %}
|
||||
{% set default_working_dir = "/app/vllm-omni" %}
|
||||
|
||||
- group: "AMD Tests"
|
||||
depends_on: ~
|
||||
steps:
|
||||
- label: "AMD: :docker: build image"
|
||||
depends_on: ~
|
||||
soft_fail: false
|
||||
commands:
|
||||
- "source .buildkite/scripts/docker_login_ecr_public.sh && safe_docker_login_ecr_public"
|
||||
- "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target final --progress plain ."
|
||||
- "docker push {{ docker_image_amd }}"
|
||||
key: "amd-build"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: -1 # Agent was lost
|
||||
limit: 1
|
||||
- exit_status: -10 # Agent was lost
|
||||
limit: 1
|
||||
- exit_status: 1 # Machine occasionally fail
|
||||
limit: 1
|
||||
agents:
|
||||
queue: cpu_queue_premerge
|
||||
|
||||
{% for step in steps %}
|
||||
{% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
|
||||
- label: "{{ step.agent_pool }}: {{ step.label }}"
|
||||
depends_on: amd-build
|
||||
agents:
|
||||
{% if step.agent_pool %}
|
||||
queue: amd_{{ step.agent_pool }}
|
||||
{% else %}
|
||||
queue: amd_mi325_1
|
||||
{% endif %}
|
||||
command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh "(command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }} ; {{ step.command or (step.commands | join(" && ")) | safe }}"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
priority: 100
|
||||
{% if step.grade and step.grade == "Blocking" %}
|
||||
soft_fail: false
|
||||
{% else %}
|
||||
soft_fail: true
|
||||
{% endif%}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
@@ -1,649 +0,0 @@
|
||||
---
|
||||
name: add-diffusion-model
|
||||
description: Add a new diffusion model (text-to-image, text-to-video, image-to-video, text-to-audio, image editing) to vLLM-Omni, including Cache-DiT acceleration and parallelism support (TP, SP/USP, CFG-Parallel, HSDP). Use when integrating a new diffusion model, porting a diffusers pipeline or a custom model repo to vllm-omni, creating a new DiT transformer adapter, adding diffusion model support, or enabling multi-GPU parallelism and cache acceleration for an existing model.
|
||||
---
|
||||
|
||||
# Adding a Diffusion Model to vLLM-Omni
|
||||
|
||||
## Overview
|
||||
|
||||
This skill guides you through adding a new diffusion model to vLLM-Omni. The model may come from HuggingFace Diffusers (structured pipeline) or from a private/custom repo. The workflow differs significantly depending on the source.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, determine:
|
||||
|
||||
1. **Model category**: Text-to-Image, Text-to-Video, Image-to-Video, Image Editing, Text-to-Audio, or Omni
|
||||
2. **Reference source**: Diffusers pipeline, custom repo, or a combination
|
||||
3. **Model HuggingFace ID** or local checkpoint path
|
||||
4. **Architecture**: Scheduler, text encoder, VAE, transformer/backbone
|
||||
|
||||
## Step 0: Classify the Migration Path
|
||||
|
||||
Check the model's HF repo for `model_index.json`. This determines your path:
|
||||
|
||||
| Scenario | How to identify | Migration path |
|
||||
|----------|----------------|----------------|
|
||||
| **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip implementation, then validate model-specific examples, tests, and docs as needed |
|
||||
| **Diffusers-based** | Has standard `model_index.json` with `_diffusers_version`, subfolders for `transformer/`, `vae/`, etc. | Follow **Path A** below |
|
||||
| **Custom/private repo** | No diffusers `model_index.json`, weights in non-standard format, custom model code in a separate git repo | Follow **Path B** below |
|
||||
| **Hybrid** | Has some diffusers components (VAE) but custom transformer/fusion | Mix of Path A and Path B |
|
||||
|
||||
## Path A: Diffusers-Based Model
|
||||
|
||||
For models with a standard diffusers layout. See [references/transformer-adaptation.md](references/transformer-adaptation.md) for detailed code patterns.
|
||||
|
||||
### A1. Analyze `model_index.json`
|
||||
|
||||
Identify components: `transformer`, `scheduler`, `vae`, `text_encoder`, `tokenizer`.
|
||||
|
||||
### A2. Create model directory
|
||||
|
||||
```
|
||||
vllm_omni/diffusion/models/your_model_name/
|
||||
├── __init__.py
|
||||
├── pipeline_your_model.py
|
||||
└── your_model_transformer.py
|
||||
```
|
||||
|
||||
### A3. Adapt transformer
|
||||
|
||||
1. Copy from diffusers source. Remove mixins (`ModelMixin`, `ConfigMixin`, `AttentionModuleMixin`).
|
||||
2. Replace attention with `vllm_omni.diffusion.attention.layer.Attention` (QKV shape: `[B, seq, heads, head_dim]`).
|
||||
3. Add `od_config: OmniDiffusionConfig | None = None` to `__init__`.
|
||||
4. Add `load_weights()` method mapping diffusers weight names to vllm-omni names.
|
||||
5. Add class attributes for acceleration features such as `_repeated_blocks` and `_layerwise_offload_blocks_attrs` (see [references/transformer-adaptation.md](references/transformer-adaptation.md) for examples).
|
||||
|
||||
### A4. Adapt pipeline
|
||||
|
||||
Inherit from `nn.Module`. The key contract:
|
||||
|
||||
```python
|
||||
class YourPipeline(nn.Module):
|
||||
def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
|
||||
# Load VAE, text encoder, tokenizer via from_pretrained()
|
||||
# Instantiate transformer (weights loaded later via weights_sources)
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=od_config.model, subfolder="transformer",
|
||||
prefix="transformer.", fall_back_to_pt=True)]
|
||||
|
||||
def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
|
||||
# Encode prompt → prepare latents → denoise loop → VAE decode
|
||||
return DiffusionOutput(output=output)
|
||||
|
||||
def load_weights(self, weights):
|
||||
return AutoWeightsLoader(self).load_weights(weights)
|
||||
```
|
||||
|
||||
Add post/pre-process functions in the same pipeline file. Register them in `registry.py`.
|
||||
|
||||
### A4.1 Add progress bar support (recommended)
|
||||
|
||||
For pipelines with a standard denoising loop, prefer the existing progress bar pattern instead of hand-rolled logging.
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
|
||||
class YourPipeline(nn.Module, ProgressBarMixin):
|
||||
def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
|
||||
# ... prepare timesteps / latents ...
|
||||
with self.progress_bar(total=len(timesteps)) as progress_bar:
|
||||
for i, t in enumerate(timesteps):
|
||||
# predict noise / scheduler step
|
||||
latents = ...
|
||||
progress_bar.update()
|
||||
|
||||
return DiffusionOutput(output=output)
|
||||
```
|
||||
|
||||
For custom loop structures, follow `vllm_omni/diffusion/models/progress_bar.py` and existing pipelines using `ProgressBarMixin`.
|
||||
|
||||
### A5. Register, test, docs → continue at Step 4 below.
|
||||
|
||||
---
|
||||
|
||||
## Path B: Custom/Private Repo Model
|
||||
|
||||
For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
|
||||
|
||||
### B1. Understand the reference repo
|
||||
|
||||
Study the original model's code to identify:
|
||||
- **Model architecture files** (transformers, fusion modules, embeddings)
|
||||
- **Weight format** (safetensors, `.pth`, custom checkpoint structure)
|
||||
- **Weight loading helpers** (custom init functions, checkpoint loaders)
|
||||
- **Pre/post-processing** (image/audio transforms, tokenization, VAE encode/decode)
|
||||
- **External dependencies** (packages not on PyPI)
|
||||
- **Config format** (JSON config files, hardcoded dicts)
|
||||
|
||||
### B2. Decide what lives WHERE
|
||||
|
||||
This is the key design decision for custom models. Follow these placement rules:
|
||||
|
||||
| Code type | Where to place | Example |
|
||||
|-----------|---------------|---------|
|
||||
| **Pipeline orchestration** (init, forward, denoise loop) | `vllm_omni/diffusion/models/<name>/pipeline_<name>.py` | Always required |
|
||||
| **Custom transformer/backbone** (ported and adapted to vllm-omni) | `vllm_omni/diffusion/models/<name>/<name>_transformer.py` or similar | `wan2_2.py`, `fusion.py`, `bagel_transformer.py` |
|
||||
| **Custom sub-models** (VAE, fusion, autoencoder) | `vllm_omni/diffusion/models/<name>/` as separate files | `autoencoder.py`, `fusion.py` |
|
||||
| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | original-repo package via git clone |
|
||||
| **Hardcoded model configs** | Module-level dicts in pipeline file | `VIDEO_CONFIG`, `AUDIO_CONFIG` dicts |
|
||||
| **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_<name>.py` |
|
||||
| **Custom `model_index.json`** | Generated by download script, placed at model root | Minimal: `{"_class_name": "YourPipeline", ...}` |
|
||||
|
||||
### B3. Handle external dependencies
|
||||
|
||||
If the model's code lives in a separate git repo:
|
||||
|
||||
**Option 1: Import with graceful fallback** (recommended for models with external utils)
|
||||
|
||||
```python
|
||||
try:
|
||||
from external_model.utils import init_vae, load_checkpoint
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"Failed to import from dependency 'external_model'. "
|
||||
"Please run the download script first."
|
||||
)
|
||||
```
|
||||
|
||||
**Option 2: Port the code directly** (preferred when feasible)
|
||||
|
||||
Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and adapt them. This avoids external dependencies. BAGEL does this — `autoencoder.py` and `bagel_transformer.py` are ported directly.
|
||||
|
||||
**Decision criteria**: Port if the code is self-contained and won't diverge. Use external deps if the model repo is actively maintained and the code is complex.
|
||||
|
||||
### B4. Handle custom weight loading
|
||||
|
||||
Custom models have two common patterns for weight loading:
|
||||
|
||||
**Pattern 1: Bypass standard loader** (eager custom init)
|
||||
|
||||
When the original model has complex custom init functions that load weights in `__init__`:
|
||||
|
||||
```python
|
||||
class CustomPipeline(nn.Module):
|
||||
def __init__(self, *, od_config, prefix=""):
|
||||
super().__init__()
|
||||
model = od_config.model
|
||||
# Load everything eagerly in __init__ using custom helpers
|
||||
self.vae = custom_init_vae(model, device=self.device)
|
||||
self.text_encoder = custom_init_text_encoder(model, device=self.device)
|
||||
self.transformer = CustomFusionModel(CONFIG)
|
||||
load_custom_checkpoint(
|
||||
self.transformer,
|
||||
checkpoint_path=os.path.join(model, "model.safetensors"),
|
||||
)
|
||||
# NO weights_sources defined — bypasses standard loader
|
||||
|
||||
def load_weights(self, weights):
|
||||
pass # No-op — all weights loaded in __init__
|
||||
```
|
||||
|
||||
**Pattern 2: Use standard loader with custom `load_weights`** (BAGEL style)
|
||||
|
||||
When weights are in safetensors format but need name remapping:
|
||||
|
||||
```python
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
class CustomPipeline(nn.Module):
|
||||
def __init__(self, *, od_config, prefix=""):
|
||||
super().__init__()
|
||||
# Instantiate model architecture without weights
|
||||
self.bagel = BagelModel(config)
|
||||
self.vae = AutoEncoder(ae_params)
|
||||
|
||||
# Point loader at the safetensors in the model root
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=od_config.model,
|
||||
subfolder=None, # weights at root, not in subfolder
|
||||
prefix="",
|
||||
fall_back_to_pt=False,
|
||||
)
|
||||
]
|
||||
|
||||
def load_weights(self, weights):
|
||||
# Custom name remapping for non-diffusers weight names
|
||||
params = dict(self.named_parameters())
|
||||
loaded = set()
|
||||
for name, tensor in weights:
|
||||
# Remap original weight names to vllm-omni module names
|
||||
name = self._remap_weight_name(name)
|
||||
if name in params:
|
||||
param = params[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, tensor)
|
||||
loaded.add(name)
|
||||
return loaded
|
||||
```
|
||||
|
||||
### B5. Create the `model_index.json`
|
||||
|
||||
Custom models need a `model_index.json` at the model root for vllm-omni to discover them. For custom models, this is minimal:
|
||||
|
||||
```json
|
||||
{
|
||||
"_class_name": "YourModelPipeline",
|
||||
"custom_key": "path/to/custom_weights.safetensors"
|
||||
}
|
||||
```
|
||||
|
||||
The `_class_name` must match a key in `_DIFFUSION_MODELS` in `registry.py`. Additional keys are model-specific (accessed via `od_config.model_config`).
|
||||
|
||||
If the model's weights come from multiple HF repos, write a **download script** that:
|
||||
1. Downloads from each repo
|
||||
2. Assembles into a single directory
|
||||
3. Generates `model_index.json`
|
||||
4. Installs any external dependencies (git clone + `.pth` file)
|
||||
|
||||
Place at: `examples/offline_inference/<name>/download_<name>.py`
|
||||
|
||||
### B6. Handle multi-modal inputs
|
||||
|
||||
If the model accepts images, audio, or other multi-modal inputs, implement the protocol classes from `vllm_omni/diffusion/models/interface.py`:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput, SupportAudioInput
|
||||
|
||||
class MyPipeline(nn.Module, SupportImageInput, SupportAudioInput):
|
||||
# Protocol markers — the engine uses these to enable proper input routing
|
||||
pass
|
||||
```
|
||||
|
||||
Preprocessing for custom models is typically done **inside `forward()`** rather than via registered pre-process functions, since the logic is often tightly coupled to the model.
|
||||
|
||||
### B7. Continue at Step 4 below.
|
||||
|
||||
---
|
||||
|
||||
## Common Steps (Both Paths)
|
||||
|
||||
### Step 4: Register Model in registry.py
|
||||
|
||||
Edit `vllm_omni/diffusion/registry.py`:
|
||||
|
||||
```python
|
||||
_DIFFUSION_MODELS = {
|
||||
"YourModelPipeline": ("your_model_name", "pipeline_your_model", "YourModelPipeline"),
|
||||
}
|
||||
_DIFFUSION_POST_PROCESS_FUNCS = {
|
||||
"YourModelPipeline": "get_your_model_post_process_func", # if applicable
|
||||
}
|
||||
_DIFFUSION_PRE_PROCESS_FUNCS = {
|
||||
"YourModelPipeline": "get_your_model_pre_process_func", # if applicable
|
||||
}
|
||||
```
|
||||
|
||||
The registry key is the `_class_name` from `model_index.json`. The tuple is `(folder_name, module_file, class_name)`.
|
||||
|
||||
Create `__init__.py` exporting the pipeline class and any factory functions.
|
||||
|
||||
### Step 5: Run, Test, Debug
|
||||
|
||||
Use the appropriate existing example script:
|
||||
|
||||
| Category | Script |
|
||||
|----------|--------|
|
||||
| Text-to-Image | `examples/offline_inference/text_to_image/text_to_image.py` |
|
||||
| Text-to-Video | `examples/offline_inference/text_to_video/text_to_video.py` |
|
||||
| Image-to-Video | `examples/offline_inference/image_to_video/image_to_video.py` |
|
||||
| Image-to-Image | `examples/offline_inference/image_to_image/image_edit.py` |
|
||||
| Text-to-Audio | `examples/offline_inference/text_to_audio/text_to_audio.py` |
|
||||
|
||||
For custom/Omni models that don't fit these categories, create a dedicated example script.
|
||||
|
||||
**Validation**: No errors, output is meaningful, quality matches reference implementation.
|
||||
|
||||
See [references/troubleshooting.md](references/troubleshooting.md) for common errors.
|
||||
|
||||
### Step 6: Add Example Scripts
|
||||
|
||||
For Omni or custom models, create:
|
||||
- `examples/offline_inference/your_model_name/` — offline script + README
|
||||
- `examples/online_serving/your_model_name/` — server script + client
|
||||
- Download script if weights require assembly from multiple sources
|
||||
|
||||
### Step 7: Update Documentation
|
||||
|
||||
Required updates:
|
||||
1. `docs/user_guide/diffusion/parallelism/overview.md` — parallelism support overview/table
|
||||
2. `docs/user_guide/diffusion/cpu_offload.md` — if CPU offload supported (add to supported models table)
|
||||
3. `docs/user_guide/diffusion/cache_acceleration/teacache.md` — if TeaCache supported
|
||||
4. `docs/user_guide/diffusion/cache_acceleration/cache_dit.md` — if Cache-DiT supported
|
||||
5. Offline example docs under `examples/offline_inference/<name>/` (`README.md` or category-specific `.md`)
|
||||
6. `examples/online_serving/<name>/README.md` — online serving docs
|
||||
|
||||
### Step 8: Add E2E Tests
|
||||
|
||||
**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md), [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md), and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md).
|
||||
|
||||
Classify the model's **CI priority** first:
|
||||
|
||||
| Priority | Required test levels | Files & markers |
|
||||
|----------|---------------------|-----------------|
|
||||
| **High** (listed in [#1832](https://github.com/vllm-project/vllm-omni/issues/1832) or on the diffusion hot path) | **L1** · **L2** online · **L3** online + offline · **L4** feature + performance | See table below |
|
||||
| **Medium** (*normal priority* in L4 docs) | **L3** online + offline · **L4** feature only | Fewer L4 parametrized rows |
|
||||
| **Low** | **L4** feature only | One or two `*_expansion.py` cases |
|
||||
|
||||
**Per-level deliverables (diffusion / `pytest.mark.diffusion`):**
|
||||
|
||||
| Level | Location | Marker | CI pipeline | Notes |
|
||||
|-------|----------|--------|-------------|-------|
|
||||
| **L1** | `tests/diffusion/models/{slug}/`, `tests/diffusion/cache/`, transformer unit tests | `core_model` + `cpu` | `test-ready.yml` | Weight remap, `_sp_plan`, cache enabler registration, shape contracts |
|
||||
| **L2** | `tests/e2e/online_serving/test_{slug}.py` (and offline if the category is offline-first) | **`core_model` + `advanced_model`** (both on baseline smoke) + `diffusion` + `@hardware_test` / `hardware_marks` | `test-ready.yml` | Default deploy smoke — minimal `num_inference_steps`, single prompt |
|
||||
| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` when offline matters | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `diffusion`) | `test-merge.yml` **or** merged into nightly diffusion function job | Real weights, streaming/API paths, LoRA/offload smoke |
|
||||
| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` with per-case **`mark`** (`hardware_marks` + `full_model` + `diffusion`) |
|
||||
|
||||
**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).
|
||||
|
||||
**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (`cards_1` vs `not cards_1`).
|
||||
|
||||
**L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.
|
||||
|
||||
**Reference implementations:** `tests/e2e/online_serving/test_qwen_image_edit_expansion.py`, `tests/e2e/online_serving/test_longcat_image_expansion.py`, `tests/e2e/online_serving/test_hunyuan_video_15_expansion.py`.
|
||||
|
||||
**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:** deploy constants, prompts, sampling dicts, and inline `request_config` / `form_data` belong in each `test_{slug}.py` and `test_{slug}_expansion.py`. Do not add per-model files under `tests/helpers/`; reuse only repo-wide harness (`mark`, `media`, `runtime`, `stage_config`, `assertions`). **L2+ online/offline e2e:** reuse or add `send_*_request` in `tests/helpers/runtime.py` — tests call the handler, not raw `omni.generate` / HTTP. See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers**.
|
||||
|
||||
### Step 9: Add Cache-DiT Acceleration
|
||||
|
||||
Cache-DiT accelerates inference by caching intermediate computation results across denoising steps. After your model is working correctly on a single GPU, add cache-dit support.
|
||||
|
||||
See [references/cache-dit-patterns.md](references/cache-dit-patterns.md) for detailed code patterns.
|
||||
|
||||
#### 9a. Determine your model type
|
||||
|
||||
| Model Type | Description | Action |
|
||||
|------------|-------------|--------|
|
||||
| **Standard single-transformer** | One transformer with one `ModuleList` of blocks | No code needed — `CacheDiTBackend` auto-detects via `enable_cache_for_dit()` |
|
||||
| **Multi-block-list** | One transformer with multiple block lists (e.g., `transformer_blocks` + `single_transformer_blocks`) | Write custom enabler with `BlockAdapter` |
|
||||
| **Dual-transformer** | Two transformers (e.g., high-noise + low-noise) | Write custom enabler with `BlockAdapter` wrapping both |
|
||||
|
||||
#### 9b. Standard models — verify automatic support
|
||||
|
||||
For standard single-transformer models, test directly:
|
||||
|
||||
```python
|
||||
omni = Omni(
|
||||
model="your-model-name",
|
||||
cache_backend="cache_dit",
|
||||
cache_config={
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
Check logs for "Cache-dit enabled successfully on xxx". If it works, skip to Step 9e.
|
||||
|
||||
#### 9c. Custom architectures — write a custom enabler
|
||||
|
||||
For multi-block-list or dual-transformer models, write a custom enabler function:
|
||||
|
||||
```python
|
||||
from cache_dit import BlockAdapter, ForwardPattern, ParamsModifier, DBCacheConfig
|
||||
|
||||
def enable_cache_for_your_model(pipeline, cache_config):
|
||||
db_cache_config = DBCacheConfig(
|
||||
num_inference_steps=None,
|
||||
Fn_compute_blocks=cache_config.Fn_compute_blocks,
|
||||
Bn_compute_blocks=cache_config.Bn_compute_blocks,
|
||||
max_warmup_steps=cache_config.max_warmup_steps,
|
||||
max_cached_steps=cache_config.max_cached_steps,
|
||||
max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
|
||||
residual_diff_threshold=cache_config.residual_diff_threshold,
|
||||
)
|
||||
|
||||
cache_dit.enable_cache(
|
||||
BlockAdapter(
|
||||
transformer=pipeline.transformer,
|
||||
blocks=[
|
||||
pipeline.transformer.transformer_blocks,
|
||||
pipeline.transformer.single_transformer_blocks,
|
||||
],
|
||||
forward_pattern=[ForwardPattern.Pattern_1, ForwardPattern.Pattern_1],
|
||||
params_modifiers=[ParamsModifier(...)],
|
||||
),
|
||||
cache_config=db_cache_config,
|
||||
)
|
||||
|
||||
def refresh_cache_context(pipeline, num_inference_steps, verbose=True):
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose
|
||||
)
|
||||
return refresh_cache_context
|
||||
```
|
||||
|
||||
#### 9d. Register the custom enabler
|
||||
|
||||
Add your enabler to `CUSTOM_DIT_ENABLERS` in
|
||||
`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
|
||||
|
||||
```python
|
||||
CUSTOM_DIT_ENABLERS = {
|
||||
"Wan22Pipeline": enable_cache_for_wan22,
|
||||
"LongCatImagePipeline": enable_cache_for_longcat_image,
|
||||
"YourModelPipeline": enable_cache_for_your_model, # Add here
|
||||
}
|
||||
```
|
||||
|
||||
#### 9e. Test Cache-DiT
|
||||
|
||||
```python
|
||||
omni = Omni(
|
||||
model="your-model-name",
|
||||
cache_backend="cache_dit",
|
||||
cache_config={
|
||||
"Fn_compute_blocks": 1, "Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4, "residual_diff_threshold": 0.24,
|
||||
}
|
||||
)
|
||||
images = omni.generate("a beautiful landscape",
|
||||
OmniDiffusionSamplingParams(num_inference_steps=50))
|
||||
```
|
||||
|
||||
**Verify**: 1) logs show cache enabled, 2) 1.5-2x speedup, 3) output quality acceptable vs baseline.
|
||||
|
||||
If quality degrades, lower `residual_diff_threshold` (try 0.12-0.18) or increase `max_warmup_steps` (try 6-8).
|
||||
|
||||
---
|
||||
|
||||
### Step 10: Add Parallelism Support
|
||||
|
||||
After the model works on a single GPU, add multi-GPU parallelism. Add each type incrementally, testing after each addition.
|
||||
|
||||
See [references/parallelism-patterns.md](references/parallelism-patterns.md) for detailed code patterns and API reference.
|
||||
|
||||
**Recommended order**: TP → SP/USP → CFG Parallel → HSDP
|
||||
|
||||
#### 10a. Tensor Parallelism (TP)
|
||||
|
||||
Shards DiT linear layers across GPUs. Requires code changes in the transformer.
|
||||
|
||||
**What to change in the transformer**:
|
||||
1. Replace `nn.Linear` with `ColumnParallelLinear` / `RowParallelLinear` / `QKVParallelLinear`
|
||||
2. Update `load_weights()` to handle QKV fusion with `stacked_params_mapping`
|
||||
3. Use `self.to_qkv.num_heads` (local heads) instead of total heads for split sizes
|
||||
|
||||
```python
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear, RowParallelLinear, ColumnParallelLinear,
|
||||
)
|
||||
|
||||
# Attention: QKV → RowParallel output
|
||||
self.to_qkv = QKVParallelLinear(dim, head_dim, num_heads, num_kv_heads)
|
||||
self.to_out = RowParallelLinear(dim, dim, input_is_parallel=True)
|
||||
|
||||
# FFN: ColumnParallel → RowParallel
|
||||
self.w1 = ColumnParallelLinear(dim, ffn_dim)
|
||||
self.w2 = RowParallelLinear(ffn_dim, dim, input_is_parallel=True)
|
||||
```
|
||||
|
||||
**Constraints**: `num_heads % tp_size == 0` and `num_kv_heads % tp_size == 0`.
|
||||
|
||||
**Test**: `--tensor-parallel-size 2`
|
||||
|
||||
#### 10b. Sequence Parallelism (SP / USP)
|
||||
|
||||
Splits sequence tokens across GPUs. Non-intrusive via `_sp_plan` on the transformer class — no changes to `forward()`.
|
||||
|
||||
**What to change in the transformer**:
|
||||
|
||||
Add `_sp_plan` class attribute:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput, SequenceParallelOutput,
|
||||
)
|
||||
|
||||
class YourTransformer(nn.Module):
|
||||
_sp_plan = {
|
||||
"blocks.0": {
|
||||
"hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
|
||||
},
|
||||
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
}
|
||||
```
|
||||
|
||||
If inline tensor ops (e.g., `torch.cat`) exist between shard/gather points, extract them into `nn.Module` submodules so hooks can intercept them.
|
||||
|
||||
For RoPE that needs splitting, add an entry for the RoPE module with `split_output=True`.
|
||||
|
||||
**Test**: `--ulysses-degree 2` (offline) or `--usp 2` (online serving)
|
||||
|
||||
#### 10c. CFG Parallel
|
||||
|
||||
Distributes positive/negative CFG branches across 2 GPUs. Requires the pipeline to inherit `CFGParallelMixin`.
|
||||
|
||||
**What to change in the pipeline**:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
|
||||
class YourPipeline(nn.Module, CFGParallelMixin):
|
||||
def diffuse(self, ...) -> torch.Tensor:
|
||||
for i, t in enumerate(timesteps):
|
||||
positive_kwargs = {...}
|
||||
negative_kwargs = {...} if do_true_cfg else None
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg, true_cfg_scale=cfg_scale,
|
||||
positive_kwargs=positive_kwargs, negative_kwargs=negative_kwargs,
|
||||
)
|
||||
latents = self.scheduler_step_maybe_with_cfg(
|
||||
noise_pred, t, latents, do_true_cfg
|
||||
)
|
||||
return latents
|
||||
```
|
||||
|
||||
Override `predict_noise()` if your transformer call is non-standard. Override `combine_cfg_noise()` for multi-output models (e.g., video + audio).
|
||||
|
||||
**Constraint**: Exactly 2 GPUs. Only for models using classifier-free guidance.
|
||||
|
||||
**Test**: `--cfg-parallel-size 2`
|
||||
|
||||
#### 10d. HSDP (Hybrid Sharded Data Parallel)
|
||||
|
||||
Shards transformer weights via PyTorch FSDP2 to reduce per-GPU VRAM. No code changes to the forward pass — just add a class attribute.
|
||||
|
||||
**What to change in the transformer**:
|
||||
|
||||
```python
|
||||
class YourTransformer(nn.Module):
|
||||
@staticmethod
|
||||
def _is_transformer_block(name: str, module) -> bool:
|
||||
return "blocks" in name and name.split(".")[-1].isdigit()
|
||||
|
||||
_hsdp_shard_conditions = [_is_transformer_block]
|
||||
```
|
||||
|
||||
**Constraint**: Cannot combine with TP. For standalone HSDP, set `hsdp_shard_size` explicitly.
|
||||
|
||||
**Test**: `--use-hsdp` or `DiffusionParallelConfig(use_hsdp=True)`
|
||||
|
||||
#### 10e. Update parallelism documentation
|
||||
|
||||
After adding parallelism support, update:
|
||||
1. `docs/user_guide/diffusion/parallelism/overview.md` — add your model to the support overview/table
|
||||
2. Record which parallelism methods are supported (USP, Ring, CFG, TP, HSDP, VAE-Patch)
|
||||
|
||||
### Step 11: Add CPU Offload Support
|
||||
|
||||
Implement `SupportsComponentDiscovery` on your pipeline class to enable
|
||||
`--enable-cpu-offload` and `--enable-layerwise-offload`. The protocol
|
||||
declares which submodules the offloader should manage:
|
||||
|
||||
```python
|
||||
from typing import ClassVar
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
|
||||
class YourPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
_dit_modules: ClassVar[list[str]] = ["transformer"]
|
||||
_encoder_modules: ClassVar[list[str]] = ["text_encoder"]
|
||||
_vae_modules: ClassVar[list[str]] = ["vae"]
|
||||
_resident_modules: ClassVar[list[str]] = [] # optional
|
||||
```
|
||||
|
||||
- `_dit_modules`: denoising submodules (kept on GPU during diffusion loop)
|
||||
- `_encoder_modules`: encoder/vision submodules (offloaded to CPU during diffusion loop)
|
||||
- `_vae_modules`: VAE(s) (handled by both sequential and layerwise backends)
|
||||
- `_resident_modules`: additional modules to pin on GPU during layerwise
|
||||
offloading (e.g. embedders, connectors). Only used by the layerwise
|
||||
backend. Optional — defaults to `[]`.
|
||||
|
||||
All attribute names support dotted paths for nested submodules
|
||||
(e.g. `"pipe.transformer"`, `"bagel.time_embedder"`).
|
||||
|
||||
Pipelines without `SupportsComponentDiscovery` fall back to scanning
|
||||
well-known attribute names (`transformer`, `text_encoder`, `vae`,
|
||||
etc.), which fails for non-standard names.
|
||||
|
||||
### Step 12: Performance Profiling
|
||||
|
||||
After verifying correctness and implementing parallelism/caching, profile the model's performance to identify bottlenecks and ensure optimal execution.
|
||||
|
||||
See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.md#3-profiling-single-stage-diffusion) guide for detailed instructions on:
|
||||
1. Using the PyTorch profiler (`profiler: "torch"`) to capture detailed CPU/CUDA traces.
|
||||
2. Using Nsight Systems (`nsys`) with `profiler: "cuda"` for low-overhead CUDA traces.
|
||||
3. Controlling profiling via `omni.start_profile()` and `omni.stop_profile()`.
|
||||
|
||||
---
|
||||
|
||||
## Pre-commit conventions
|
||||
|
||||
New library files must pass the local gates in
|
||||
[docs/contributing/README.md](../../../docs/contributing/README.md#linting).
|
||||
That page is the full hook list (SPDX, forbidden imports including Hugging Face
|
||||
Hub / Triton / pickle, `torch.cuda`, mypy, test marks, markdownlint, Buildkite,
|
||||
shellcheck). In particular:
|
||||
|
||||
- SPDX copyright is `vLLM-Omni project` (stale `vLLM project` is rewritten).
|
||||
- Use `import regex as re` and `pybase64` in `vllm_omni/`; do not import stdlib
|
||||
`re` or `base64`. Hugging Face Hub downloads go through
|
||||
`vllm.transformers_utils.repo_utils`.
|
||||
- Do not add `torch.cuda.*` call sites; use `current_omni_platform`.
|
||||
- New `tests/**/test_*.py` files need a CI level mark and a hardware mark.
|
||||
- Do not expand `CHECK_IMPORTS[*].allowed_files` or `ALLOWED_FILES` without review.
|
||||
- GitHub Actions skips SPDX/shellcheck/mypy-3.10/test-marks/markdownlint; run
|
||||
`pre-commit` locally.
|
||||
|
||||
## Iterative Development Tips
|
||||
|
||||
1. **Start minimal**: Basic generation first, no parallelism/caching
|
||||
2. **Use `--enforce-eager`**: Disable torch.compile during debugging
|
||||
3. **Use small models**: Test with smaller variants first
|
||||
4. **Check tensor shapes**: Most errors are reshape mismatches in attention
|
||||
5. **Add features incrementally**: Single GPU → TP → SP → CFG → HSDP → Cache-DiT
|
||||
6. **For custom models**: Get the model running with the original code first, then progressively replace components with vllm-omni equivalents
|
||||
7. **Cache-DiT before parallelism tuning**: Cache-DiT is lossy — verify quality at baseline before combining with parallelism
|
||||
8. **Combine lossless + lossy**: e.g., TP + SP + Cache-DiT for maximum throughput
|
||||
|
||||
## Reference Files
|
||||
|
||||
- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
|
||||
- [Transformer Adaptation](references/transformer-adaptation.md) — porting transformers from diffusers
|
||||
- [Custom Model Patterns](references/custom-model-patterns.md) — patterns for non-diffusers models
|
||||
- [Parallelism Patterns](references/parallelism-patterns.md) — TP, SP/USP, CFG parallel, HSDP implementation details
|
||||
- [Cache-DiT Patterns](references/cache-dit-patterns.md) — cache-dit acceleration for standard and custom architectures
|
||||
- [Troubleshooting](references/troubleshooting.md) — common errors and fixes
|
||||
@@ -1,260 +0,0 @@
|
||||
# Cache-DiT Patterns Reference
|
||||
|
||||
## Overview
|
||||
|
||||
Cache-DiT accelerates Diffusion Transformers by caching intermediate computation results across denoising steps. Adjacent steps produce similar features, so redundant computations can be skipped.
|
||||
|
||||
Three caching strategies:
|
||||
- **DBCache**: Dynamic block-level caching — selectively computes or caches transformer blocks based on residual differences
|
||||
- **TaylorSeer**: Calibration-based prediction using Taylor expansion to estimate block outputs
|
||||
- **SCM** (Step Computation Masking): Dynamic step skipping based on configurable policies
|
||||
|
||||
**Typical speedup**: 1.5-2.5x depending on model and configuration.
|
||||
|
||||
**Official docs**: https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cache_dit
|
||||
|
||||
## Architecture
|
||||
|
||||
vLLM-Omni integrates cache-dit through `CacheDiTBackend`:
|
||||
|
||||
| Component | Purpose |
|
||||
|-----------|---------|
|
||||
| `CacheDiTBackend` | Unified backend — auto-selects enabler (standard or custom) |
|
||||
| `enable_cache_for_dit()` | Default enabler for standard single-transformer models |
|
||||
| `CUSTOM_DIT_ENABLERS` dict | Registry of custom enablers keyed by pipeline class name |
|
||||
| `BlockAdapter` | Wraps complex architectures (multi-block-list or multi-transformer) |
|
||||
| `ForwardPattern` | Specifies block forward signature: `Pattern_0`, `Pattern_1`, `Pattern_2` |
|
||||
| `ParamsModifier` | Per-transformer or per-block-list config customization |
|
||||
| `DBCacheConfig` | Configuration for DBCache parameters |
|
||||
| `cache_dit.refresh_context()` | Updates cache context when `num_inference_steps` changes |
|
||||
|
||||
**Source files**:
|
||||
- `vllm_omni/diffusion/cache/cachedit/__init__.py` — public package API
|
||||
- `vllm_omni/diffusion/cache/cachedit/backend.py` — generic backend lifecycle and integration
|
||||
- `vllm_omni/diffusion/cache/cachedit/config.py` — Cache-DiT configuration types
|
||||
- `vllm_omni/diffusion/cache/cachedit/model_specific.py` — model-specific adapters, enablers, and registry
|
||||
|
||||
Outside `vllm_omni/diffusion/cache`, import Cache-DiT symbols only from
|
||||
`vllm_omni.diffusion.cache.cachedit`.
|
||||
|
||||
## Standard Models: Automatic Support
|
||||
|
||||
Most DiT models follow this pattern:
|
||||
- Single transformer with one `nn.ModuleList` of blocks
|
||||
- Standard forward signature
|
||||
- Compatible with cache-dit's automatic detection
|
||||
|
||||
**Examples**: Qwen-Image, Z-Image, FLUX
|
||||
|
||||
No code changes needed. `CacheDiTBackend` automatically uses `enable_cache_for_dit()`:
|
||||
|
||||
```python
|
||||
from vllm_omni import Omni
|
||||
|
||||
omni = Omni(
|
||||
model="Qwen/Qwen-Image",
|
||||
cache_backend="cache_dit",
|
||||
cache_config={
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
What happens automatically:
|
||||
|
||||
```python
|
||||
def enable_cache_for_dit(pipeline, cache_config):
|
||||
db_cache_config = DBCacheConfig(
|
||||
num_inference_steps=None,
|
||||
Fn_compute_blocks=cache_config.Fn_compute_blocks,
|
||||
Bn_compute_blocks=cache_config.Bn_compute_blocks,
|
||||
max_warmup_steps=cache_config.max_warmup_steps,
|
||||
max_cached_steps=cache_config.max_cached_steps,
|
||||
max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
|
||||
residual_diff_threshold=cache_config.residual_diff_threshold,
|
||||
)
|
||||
|
||||
cache_dit.enable_cache(pipeline.transformer, cache_config=db_cache_config)
|
||||
|
||||
def refresh_cache_context(pipeline, num_inference_steps, verbose=True):
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose
|
||||
)
|
||||
return refresh_cache_context
|
||||
```
|
||||
|
||||
## Custom Architectures: Writing Custom Enablers
|
||||
|
||||
### When you need a custom enabler
|
||||
|
||||
- Model has multiple block lists in one transformer (e.g., `transformer_blocks` + `single_transformer_blocks`)
|
||||
- Model has two transformers (e.g., high-noise + low-noise like Wan2.2)
|
||||
- Model uses non-standard block forward signature
|
||||
|
||||
### Pattern 1: Multi-Block-List (LongCat-Image style)
|
||||
|
||||
Single transformer with two block lists:
|
||||
|
||||
```python
|
||||
import cache_dit
|
||||
from cache_dit import BlockAdapter, ForwardPattern, ParamsModifier, DBCacheConfig
|
||||
|
||||
def enable_cache_for_your_model(pipeline, cache_config):
|
||||
db_cache_config = DBCacheConfig(
|
||||
num_inference_steps=None,
|
||||
Fn_compute_blocks=cache_config.Fn_compute_blocks,
|
||||
Bn_compute_blocks=cache_config.Bn_compute_blocks,
|
||||
max_warmup_steps=cache_config.max_warmup_steps,
|
||||
max_cached_steps=cache_config.max_cached_steps,
|
||||
max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
|
||||
residual_diff_threshold=cache_config.residual_diff_threshold,
|
||||
)
|
||||
|
||||
cache_dit.enable_cache(
|
||||
BlockAdapter(
|
||||
transformer=pipeline.transformer,
|
||||
blocks=[
|
||||
pipeline.transformer.transformer_blocks,
|
||||
pipeline.transformer.single_transformer_blocks,
|
||||
],
|
||||
forward_pattern=[ForwardPattern.Pattern_1, ForwardPattern.Pattern_1],
|
||||
params_modifiers=[ParamsModifier(...)],
|
||||
),
|
||||
cache_config=db_cache_config,
|
||||
)
|
||||
|
||||
def refresh_cache_context(pipeline, num_inference_steps, verbose=True):
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose
|
||||
)
|
||||
return refresh_cache_context
|
||||
```
|
||||
|
||||
For single transformer with multiple block lists, `refresh_context` works the same as standard models — call it once on the transformer.
|
||||
|
||||
### Pattern 2: Dual-Transformer (Wan2.2 style)
|
||||
|
||||
Two transformers with separate configs:
|
||||
|
||||
```python
|
||||
def enable_cache_for_dual_transformer(pipeline, cache_config):
|
||||
db_cache_config = DBCacheConfig(...)
|
||||
|
||||
cache_dit.enable_cache(
|
||||
BlockAdapter(
|
||||
transformer=[pipeline.transformer, pipeline.transformer_2],
|
||||
blocks=[pipeline.transformer.blocks, pipeline.transformer_2.blocks],
|
||||
forward_pattern=[ForwardPattern.Pattern_2, ForwardPattern.Pattern_2],
|
||||
params_modifiers=[
|
||||
ParamsModifier(...), # Config for transformer 1
|
||||
ParamsModifier(...), # Config for transformer 2
|
||||
],
|
||||
),
|
||||
cache_config=db_cache_config,
|
||||
)
|
||||
|
||||
def refresh_cache_context(pipeline, num_inference_steps, verbose=True):
|
||||
high_steps, low_steps = _split_inference_steps(num_inference_steps)
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer, num_inference_steps=high_steps, verbose=verbose
|
||||
)
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer_2, num_inference_steps=low_steps, verbose=verbose
|
||||
)
|
||||
return refresh_cache_context
|
||||
```
|
||||
|
||||
Key difference: `refresh_context` must be called on **each transformer separately** with its own step count.
|
||||
|
||||
### Choosing the ForwardPattern
|
||||
|
||||
| Pattern | Block forward signature | Example models |
|
||||
|---------|------------------------|----------------|
|
||||
| `Pattern_0` | `block(hidden_states, **kwargs)` → residual added inside block | Default |
|
||||
| `Pattern_1` | `block(hidden_states, **kwargs)` → returns `(hidden_states, ...)` tuple | FLUX-style single blocks |
|
||||
| `Pattern_2` | `block(hidden_states, **kwargs)` → `(hidden_states, ...)` with different residual pattern | Wan2.2 blocks |
|
||||
|
||||
Inspect your block's `forward()` return type and residual connection pattern to choose the right one. See [Cache-DiT API Reference](https://cache-dit.readthedocs.io/en/latest/user_guide/CACHE_API/) for details.
|
||||
|
||||
## Registering Custom Enablers
|
||||
|
||||
Add your enabler to `CUSTOM_DIT_ENABLERS` in
|
||||
`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
|
||||
|
||||
```python
|
||||
CUSTOM_DIT_ENABLERS = {
|
||||
"Wan22Pipeline": enable_cache_for_wan22,
|
||||
"LongCatImagePipeline": enable_cache_for_longcat_image,
|
||||
"YourModelPipeline": enable_cache_for_your_model,
|
||||
}
|
||||
```
|
||||
|
||||
The key must match `pipeline.__class__.__name__`.
|
||||
|
||||
## Configuration Parameters
|
||||
|
||||
| Parameter | Default | Description |
|
||||
|-----------|---------|-------------|
|
||||
| `Fn_compute_blocks` | 1 | Number of blocks to always compute at the front |
|
||||
| `Bn_compute_blocks` | 0 | Number of blocks to always compute at the back |
|
||||
| `max_warmup_steps` | 4 | Steps to run without caching at the beginning |
|
||||
| `max_cached_steps` | — | Max total cached steps |
|
||||
| `max_continuous_cached_steps` | — | Max consecutive cached steps |
|
||||
| `residual_diff_threshold` | 0.24 | Threshold for deciding whether to cache a block |
|
||||
|
||||
### Tuning for quality vs speed
|
||||
|
||||
| Goal | Adjustments |
|
||||
|------|-------------|
|
||||
| **More speed, acceptable quality loss** | Higher `residual_diff_threshold` (0.24-0.4), lower `max_warmup_steps` (2-4) |
|
||||
| **Better quality, less speed** | Lower `residual_diff_threshold` (0.12-0.18), higher `max_warmup_steps` (6-8), lower `max_continuous_cached_steps` (2) |
|
||||
|
||||
## Testing
|
||||
|
||||
```python
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
omni = Omni(
|
||||
model="your-model-name",
|
||||
cache_backend="cache_dit",
|
||||
cache_config={
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
}
|
||||
)
|
||||
images = omni.generate(
|
||||
"a beautiful landscape",
|
||||
OmniDiffusionSamplingParams(num_inference_steps=50),
|
||||
)
|
||||
```
|
||||
|
||||
CLI (online serving):
|
||||
|
||||
```bash
|
||||
vllm serve your-model --omni --port 8098 \
|
||||
--cache-backend cache_dit \
|
||||
--cache-config '{"Fn_compute_blocks": 1, "Bn_compute_blocks": 0, "max_warmup_steps": 4}'
|
||||
```
|
||||
|
||||
**Verification checklist**:
|
||||
1. Logs show "Cache-dit enabled successfully on xxx"
|
||||
2. Performance: 1.5-2x speedup vs no cache
|
||||
3. Quality: compare output with `cache_backend=None`
|
||||
|
||||
## Excluded Models
|
||||
|
||||
Models listed in `_NO_CACHE_ACCELERATION` in `vllm_omni/diffusion/registry.py` do not support cache-dit (e.g., `NextStep11Pipeline`, `StableDiffusionPipeline`). Check this set before attempting to enable cache-dit.
|
||||
|
||||
## Reference Implementations
|
||||
|
||||
| Model | Path | Notes |
|
||||
|-------|------|-------|
|
||||
| Standard DiT | `cachedit.backend::enable_cache_for_dit` | Default enabler, automatic |
|
||||
| Wan2.2 | `cachedit.model_specific::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
|
||||
| LongCat | `cachedit.config::CacheDiTAdapterConfig` | Declarative multi-block-list adapter |
|
||||
| BAGEL | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter |
|
||||
@@ -1,272 +0,0 @@
|
||||
# Custom Model Patterns Reference
|
||||
|
||||
Patterns for adding models that don't come from the standard diffusers pipeline format.
|
||||
|
||||
## Directory Structure Comparison
|
||||
|
||||
### Diffusers-based model (e.g., Wan2.2)
|
||||
|
||||
```
|
||||
vllm_omni/diffusion/models/wan2_2/
|
||||
├── __init__.py # Exports pipeline + transformer + helpers
|
||||
├── pipeline_wan2_2.py # Pipeline: loads components via from_pretrained()
|
||||
├── pipeline_wan2_2_i2v.py # Variant pipeline for image-to-video
|
||||
└── wan2_2_transformer.py # Transformer: ported from diffusers, uses Attention layer
|
||||
```
|
||||
|
||||
The transformer is loaded separately via `weights_sources` + `load_weights()`. Non-transformer components (VAE, text encoder) are loaded in `__init__` via `from_pretrained()`.
|
||||
|
||||
### Custom model with external deps
|
||||
|
||||
```
|
||||
vllm_omni/diffusion/models/<name>/
|
||||
├── __init__.py # Exports pipeline only
|
||||
├── pipeline_<name>.py # Pipeline: loads ALL weights in __init__ via custom helpers
|
||||
└── <backbone>.py # Custom transformer / fusion architecture
|
||||
|
||||
examples/offline_inference/<task>/
|
||||
└── download_<name>.py # Downloads weights and/or clones code repo
|
||||
```
|
||||
|
||||
All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency imported with try/except.
|
||||
|
||||
### Custom model with ported code (e.g., BAGEL)
|
||||
|
||||
```
|
||||
vllm_omni/diffusion/models/bagel/
|
||||
├── __init__.py
|
||||
├── pipeline_bagel.py # Pipeline: instantiates models, uses weights_sources
|
||||
├── bagel_transformer.py # Full LLM backbone (Qwen2-MoT) ported into vllm-omni
|
||||
└── autoencoder.py # Custom VAE ported from original repo
|
||||
```
|
||||
|
||||
Model code is fully ported (no external dependency). Uses `weights_sources` and `load_weights()` with custom name remapping to handle non-diffusers safetensors format.
|
||||
|
||||
## Weight Loading Patterns
|
||||
|
||||
### Pattern 1: Standard diffusers flow (Wan2.2, Z-Image, FLUX)
|
||||
|
||||
```
|
||||
init → create transformer (empty) → set weights_sources → [loader calls load_weights()]
|
||||
```
|
||||
|
||||
- `weights_sources` points to safetensors in HF subfolder (e.g., `transformer/`)
|
||||
- `load_weights()` receives `(name, tensor)` pairs from the loader
|
||||
- Name remapping handles diffusers→vllm-omni differences (QKV fusion, Sequential index removal)
|
||||
|
||||
### Pattern 2: Custom safetensors at root (BAGEL)
|
||||
|
||||
```
|
||||
init → create all models (empty) → set weights_sources(subfolder=None) → [loader calls load_weights()]
|
||||
```
|
||||
|
||||
- `weights_sources` points to **root** of model directory, not a subfolder
|
||||
- Weights have non-diffusers names (e.g., `bagel.language_model.model.layers.0.self_attn.q_proj.weight`)
|
||||
- `load_weights()` does heavy name normalization
|
||||
|
||||
```python
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=od_config.model,
|
||||
subfolder=None, # root directory
|
||||
prefix="", # no prefix stripping
|
||||
fall_back_to_pt=False,
|
||||
)
|
||||
]
|
||||
```
|
||||
|
||||
### Pattern 3: Fully custom loading
|
||||
|
||||
```
|
||||
init → load ALL weights eagerly via custom helpers → load_weights() = no-op
|
||||
```
|
||||
|
||||
- No `weights_sources` attribute — standard loader finds nothing to iterate
|
||||
- Custom init functions (e.g., `init_wan_vae_2_2()`, `load_fusion_checkpoint()`) handle downloading and loading
|
||||
- `load_weights()` is `pass`
|
||||
- Weights may come from multiple HF repos in different formats (`.pth`, `.safetensors`)
|
||||
|
||||
Use this when:
|
||||
- The original model has complex, well-tested loading code you don't want to rewrite
|
||||
- Weights span multiple HF repos
|
||||
- Weight format is non-standard (e.g., a single `.pth` file, not sharded safetensors)
|
||||
|
||||
## model_index.json for Custom Models
|
||||
|
||||
Standard diffusers `model_index.json`:
|
||||
```json
|
||||
{
|
||||
"_class_name": "WanPipeline",
|
||||
"_diffusers_version": "0.35.0.dev0",
|
||||
"scheduler": ["diffusers", "UniPCMultistepScheduler"],
|
||||
"transformer": ["diffusers", "WanTransformer3DModel"],
|
||||
"vae": ["diffusers", "AutoencoderKLWan"]
|
||||
}
|
||||
```
|
||||
|
||||
Custom model `model_index.json` (minimal):
|
||||
```json
|
||||
{
|
||||
"_class_name": "YourCustomPipeline",
|
||||
"fusion": "weights/model.safetensors"
|
||||
}
|
||||
```
|
||||
|
||||
The only **required** field is `_class_name` — it must match a key in `_DIFFUSION_MODELS` in `registry.py`. Other fields are model-specific and accessible via `od_config.model_config` dict.
|
||||
|
||||
## External Dependency Management
|
||||
|
||||
### Git clone + .pth injection
|
||||
|
||||
```python
|
||||
def download_dependency():
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with open(LOCK_FILE, "w") as f:
|
||||
fcntl.flock(f, fcntl.LOCK_EX)
|
||||
if not DEPENDENCY_DIR.exists():
|
||||
subprocess.run([
|
||||
"git", "clone", "--depth", "1",
|
||||
REPO_URL, "--branch", BRANCH,
|
||||
str(DEPENDENCY_DIR)
|
||||
], check=True)
|
||||
fcntl.flock(f, fcntl.LOCK_UN)
|
||||
|
||||
# Add to Python path via .pth file
|
||||
site_packages = Path(site.getsitepackages()[0])
|
||||
pth_file = site_packages / "vllm_omni_dependency.pth"
|
||||
pth_file.write_text(str(DEPENDENCY_DIR))
|
||||
```
|
||||
|
||||
### Direct port (BAGEL pattern)
|
||||
|
||||
Copy essential files from the original repo into `vllm_omni/diffusion/models/<name>/`. Adapt imports to use vllm-omni utilities. Benefits: no external dependency, no git clone step. Drawback: must maintain the ported code.
|
||||
|
||||
## Multi-Modal Input/Output Protocols
|
||||
|
||||
Custom models that handle images, audio, or video I/O should implement protocol classes:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.models.interface import (
|
||||
SupportImageInput, # Model accepts image input
|
||||
SupportAudioInput, # Model accepts audio input
|
||||
SupportAudioOutput, # Model produces audio output
|
||||
)
|
||||
|
||||
class MyPipeline(nn.Module, SupportImageInput, SupportAudioInput, SupportAudioOutput):
|
||||
pass # Protocol markers enable proper engine routing
|
||||
```
|
||||
|
||||
The engine checks `isinstance(pipeline, SupportImageInput)` at startup to configure input validation and warmup behavior.
|
||||
|
||||
## Hardcoded Config vs Config Files
|
||||
|
||||
Diffusers models use `config.json` in each subfolder. Custom models often use:
|
||||
|
||||
**Module-level config dicts**:
|
||||
```python
|
||||
VIDEO_CONFIG = {
|
||||
"patch_size": [1, 2, 2], "model_type": "ti2v",
|
||||
"dim": 3072, "ffn_dim": 14336, "num_heads": 24, "num_layers": 30, ...
|
||||
}
|
||||
```
|
||||
|
||||
**Loaded from custom JSON** (BAGEL):
|
||||
```python
|
||||
cfg_path = os.path.join(model_path, "config.json")
|
||||
with open(cfg_path) as f:
|
||||
bagel_cfg = json.load(f)
|
||||
vae_cfg = bagel_cfg.get("vae_config", {})
|
||||
```
|
||||
|
||||
## Custom Architecture Patterns
|
||||
|
||||
### Split forward API
|
||||
|
||||
When a fusion model needs to interleave blocks from two backbones:
|
||||
|
||||
```python
|
||||
class WanModel(nn.Module):
|
||||
def prepare_transformer_block_kwargs(self, x, t, context, ...):
|
||||
# Patch embed, time embed, text embed, RoPE
|
||||
return x, e, kwargs
|
||||
|
||||
def post_transformer_block_out(self, x, grid_sizes, e):
|
||||
# Output projection, unpatchify
|
||||
return output
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
raise NotImplementedError # Fusion model handles block iteration
|
||||
```
|
||||
|
||||
The `FusionModel` then iterates blocks in lock-step:
|
||||
```python
|
||||
for video_block, audio_block in zip(self.video_model.blocks, self.audio_model.blocks):
|
||||
video_out = video_block(video_hidden, ...)
|
||||
audio_out = audio_block(audio_hidden, ...)
|
||||
# Cross-attend between modalities
|
||||
video_out = cross_attention(video_out, audio_out)
|
||||
audio_out = cross_attention(audio_out, video_out)
|
||||
```
|
||||
|
||||
### LLM-as-denoiser (BAGEL)
|
||||
|
||||
When the backbone is a language model that also does diffusion:
|
||||
|
||||
```python
|
||||
class BagelModel(nn.Module):
|
||||
def __init__(self):
|
||||
self.language_model = Qwen2MoTForCausalLM(config)
|
||||
self.vit_model = SiglipVisionModel(vit_config)
|
||||
```
|
||||
|
||||
The LLM processes both text tokens and latent image tokens in a single forward pass, using KV caching for the text portion.
|
||||
|
||||
## Pre/Post Processing for Custom Models
|
||||
|
||||
Custom models typically handle pre/post processing **inside `forward()`** rather than via registered functions, because the logic is tightly coupled:
|
||||
|
||||
```python
|
||||
def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
|
||||
# Inline preprocessing
|
||||
image = self._load_and_resize_image(req.prompts[0].get("multi_modal_data", {}).get("image"))
|
||||
image_latent = self._vae_encode(image)
|
||||
|
||||
# ... denoising loop ...
|
||||
|
||||
# Inline postprocessing
|
||||
pil_image = self._decode_to_pil(latents)
|
||||
return DiffusionOutput(output=[pil_image])
|
||||
```
|
||||
|
||||
If pre/post functions are not registered in `_DIFFUSION_PRE_PROCESS_FUNCS` / `_DIFFUSION_POST_PROCESS_FUNCS`, the engine simply skips those steps.
|
||||
|
||||
## Download Script Template
|
||||
|
||||
```python
|
||||
# examples/offline_inference/<name>/download_<name>.py
|
||||
from huggingface_hub import snapshot_download
|
||||
import json, os
|
||||
|
||||
def main(output_dir):
|
||||
# Download model weights from HF
|
||||
snapshot_download(repo_id="org/model-weights", local_dir=os.path.join(output_dir, "weights"))
|
||||
|
||||
# Download additional components if from separate repos
|
||||
snapshot_download(repo_id="org/vae-weights", local_dir=os.path.join(output_dir, "vae"),
|
||||
allow_patterns=["*.safetensors"])
|
||||
|
||||
# Generate model_index.json
|
||||
config = {"_class_name": "YourPipeline", "custom_key": "weights/model.safetensors"}
|
||||
with open(os.path.join(output_dir, "model_index.json"), "w") as f:
|
||||
json.dump(config, f, indent=2)
|
||||
|
||||
# Install external code dependency (if needed)
|
||||
download_dependency()
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output-dir", default="./your_model")
|
||||
args = parser.parse_args()
|
||||
main(args.output_dir)
|
||||
```
|
||||
@@ -1,572 +0,0 @@
|
||||
# Parallelism Patterns Reference
|
||||
|
||||
## Overview
|
||||
|
||||
vLLM-Omni supports multiple parallelism strategies for diffusion models. Each targets a different bottleneck:
|
||||
|
||||
| Strategy | Splits | Best For | Constraint |
|
||||
|----------|--------|----------|------------|
|
||||
| Tensor Parallel (TP) | Model layers across GPUs | Latency reduction, large models | Requires fast GPU interconnect, `num_heads % tp == 0` |
|
||||
| Sequence Parallel (SP/USP) | Sequence tokens across GPUs | Long sequences (video, high-res) | Near-linear scaling |
|
||||
| CFG Parallel | Positive/negative CFG branches | Models using classifier-free guidance | Exactly 2 GPUs |
|
||||
| HSDP | Weight shards via FSDP2 | VRAM reduction | Cannot combine with TP |
|
||||
| VAE Patch Parallel | VAE decode spatial tiles | Large VAE outputs | Auto-enables tiling |
|
||||
|
||||
**Recommended integration order**: TP → SP → CFG Parallel → HSDP
|
||||
|
||||
**Official design docs**:
|
||||
- TP: https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/tensor_parallel
|
||||
- SP: https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/sequence_parallel
|
||||
- CFG: https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cfg_parallel
|
||||
- HSDP: https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/hsdp
|
||||
|
||||
---
|
||||
|
||||
## Tensor Parallelism (TP)
|
||||
|
||||
Replace standard `nn.Linear` with vLLM's parallel linear layers. This is the most invasive change but provides direct VRAM savings and compute speedup.
|
||||
|
||||
### Layer replacement rules
|
||||
|
||||
| Pattern | vLLM Layer | When to Use |
|
||||
|---------|-----------|-------------|
|
||||
| Fan-out (first in FFN) | `ColumnParallelLinear` | Projection that splits output across ranks |
|
||||
| Fan-in (second in FFN) | `RowParallelLinear` | Projection that gathers across ranks |
|
||||
| QKV projection | `QKVParallelLinear` | Fused Q/K/V for self-attention |
|
||||
| Single Q or K or V | `ColumnParallelLinear` | Separate projections (cross-attention) |
|
||||
| Attention output | `RowParallelLinear` | Output projection after attention |
|
||||
| Must not shard | `ReplicatedLinear` | Layers that must stay replicated |
|
||||
|
||||
### MLP Block (Up-Down Pattern)
|
||||
|
||||
```python
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear, RowParallelLinear,
|
||||
)
|
||||
|
||||
class TPFeedForward(nn.Module):
|
||||
def __init__(self, dim, ffn_dim):
|
||||
super().__init__()
|
||||
self.fc1 = ColumnParallelLinear(dim, ffn_dim, bias=False, return_bias=False)
|
||||
self.fc2 = RowParallelLinear(
|
||||
ffn_dim, dim, bias=False,
|
||||
input_is_parallel=True, # Input already sharded from fc1
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x, _ = self.fc1(x)
|
||||
x = torch.nn.functional.gelu(x)
|
||||
x, _ = self.fc2(x)
|
||||
return x
|
||||
```
|
||||
|
||||
### Attention Block (QKV-Out Pattern)
|
||||
|
||||
```python
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
|
||||
class TPSelfAttention(nn.Module):
|
||||
def __init__(self, dim, num_heads, num_kv_heads=None):
|
||||
super().__init__()
|
||||
num_kv_heads = num_kv_heads or num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
|
||||
self.to_qkv = QKVParallelLinear(
|
||||
hidden_size=dim,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=num_heads,
|
||||
total_num_kv_heads=num_kv_heads,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
)
|
||||
self.to_out = RowParallelLinear(
|
||||
dim, dim, bias=False,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
)
|
||||
self.attn = Attention(
|
||||
num_heads=self.to_qkv.num_heads, # Local heads per GPU
|
||||
head_size=self.head_dim,
|
||||
softmax_scale=1.0 / (self.head_dim ** 0.5),
|
||||
causal=False,
|
||||
num_kv_heads=self.to_qkv.num_kv_heads, # Local KV heads per GPU
|
||||
role="self",
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
qkv, _ = self.to_qkv(x)
|
||||
q, k, v = qkv.split(
|
||||
[self.to_qkv.num_heads * self.head_dim,
|
||||
self.to_qkv.num_kv_heads * self.head_dim,
|
||||
self.to_qkv.num_kv_heads * self.head_dim],
|
||||
dim=-1,
|
||||
)
|
||||
B, S, _ = x.shape
|
||||
q = q.view(B, S, self.to_qkv.num_heads, self.head_dim)
|
||||
k = k.view(B, S, self.to_qkv.num_kv_heads, self.head_dim)
|
||||
v = v.view(B, S, self.to_qkv.num_kv_heads, self.head_dim)
|
||||
out = self.attn(q, k, v)
|
||||
out = out.reshape(B, S, -1)
|
||||
out, _ = self.to_out(out)
|
||||
return out
|
||||
```
|
||||
|
||||
### QKV Fusion in load_weights
|
||||
|
||||
When you fuse separate Q/K/V into `QKVParallelLinear`, map diffusers' separate weight names:
|
||||
|
||||
```python
|
||||
stacked_params_mapping = [
|
||||
("to_qkv", "to_q", "q"),
|
||||
("to_qkv", "to_k", "k"),
|
||||
("to_qkv", "to_v", "v"),
|
||||
]
|
||||
|
||||
def load_weights(self, weights):
|
||||
params = dict(self.named_parameters())
|
||||
loaded = set()
|
||||
for name, tensor in weights:
|
||||
for fused_name, orig_name, shard_id in stacked_params_mapping:
|
||||
if orig_name in name:
|
||||
name = name.replace(orig_name, fused_name)
|
||||
param = params[name]
|
||||
param.weight_loader(param, tensor, shard_id)
|
||||
loaded.add(name)
|
||||
break
|
||||
else:
|
||||
if name in params:
|
||||
param = params[name]
|
||||
if hasattr(param, "weight_loader"):
|
||||
param.weight_loader(param, tensor)
|
||||
else:
|
||||
default_weight_loader(param, tensor)
|
||||
loaded.add(name)
|
||||
return loaded
|
||||
```
|
||||
|
||||
### RMSNorm with TP
|
||||
|
||||
When RMSNorm sits between TP-sharded dimensions, use `DistributedRMSNorm` — it computes global RMS via all-reduce across TP ranks. See the Wan2.2 implementation for the pattern.
|
||||
|
||||
### TP Constraints
|
||||
|
||||
- `num_heads % tp_size == 0`
|
||||
- `num_kv_heads % tp_size == 0`
|
||||
- Use `self.to_qkv.num_heads` (local per-GPU count), not total heads, for split sizes
|
||||
|
||||
### Testing TP
|
||||
|
||||
```bash
|
||||
python text_to_image.py --model Your-org/your-model \
|
||||
--tensor-parallel-size 2 --output "tp_test.png"
|
||||
```
|
||||
|
||||
**Verify**: speedup, memory reduction proportional to TP size, quality matches single-GPU.
|
||||
|
||||
### Reference implementations
|
||||
|
||||
| Model | Path |
|
||||
|-------|------|
|
||||
| Z-Image | `vllm_omni/diffusion/models/z_image/z_image_transformer.py` |
|
||||
| FLUX | `vllm_omni/diffusion/models/flux/flux_transformer.py` |
|
||||
| Qwen-Image | `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` |
|
||||
|
||||
---
|
||||
|
||||
## Sequence Parallelism (SP / USP)
|
||||
|
||||
SP splits sequence tokens across GPUs using Ulysses (all-to-all) or Ring (P2P) communication. It is applied non-intrusively via the `_sp_plan` dict — no changes to `forward()` logic.
|
||||
|
||||
### Approach 1: Non-Intrusive `_sp_plan` (Recommended)
|
||||
|
||||
The framework automatically registers hooks to shard inputs and gather outputs at `nn.Module` boundaries.
|
||||
|
||||
#### Step 1: Identify module boundaries
|
||||
|
||||
Find where tensors need sharding/gathering:
|
||||
|
||||
```python
|
||||
class MyTransformer(nn.Module):
|
||||
def __init__(self):
|
||||
self.patch_embed = PatchEmbed() # Before blocks
|
||||
self.pos_embed = RoPE() # RoPE may need splitting
|
||||
self.blocks = nn.ModuleList([...]) # Blocks process sharded x
|
||||
self.norm_out = LayerNorm()
|
||||
self.proj_out = Linear() # Gather after this
|
||||
|
||||
def forward(self, x):
|
||||
x = self.patch_embed(x)
|
||||
pos = self.pos_embed(x)
|
||||
for block in self.blocks:
|
||||
x = block(x, pos)
|
||||
x = self.norm_out(x)
|
||||
return self.proj_out(x)
|
||||
```
|
||||
|
||||
#### Step 2: Handle inline operations
|
||||
|
||||
`_sp_plan` hooks only work at `nn.Module` boundaries. Inline ops like `torch.cat()` must be extracted into submodules:
|
||||
|
||||
```python
|
||||
# BAD: Inline — hooks can't intercept
|
||||
unified = torch.cat([x, cap_feats], dim=1)
|
||||
|
||||
# GOOD: Extract into submodule
|
||||
class UnifiedPrepare(nn.Module):
|
||||
def forward(self, x, cap_feats):
|
||||
return torch.cat([x, cap_feats], dim=1)
|
||||
|
||||
self.unified_prepare = UnifiedPrepare()
|
||||
unified = self.unified_prepare(x, cap_feats)
|
||||
```
|
||||
|
||||
Common cases: `torch.cat()`, `pad_sequence()`, `tensor.reshape()`, complex preprocessing.
|
||||
|
||||
#### Step 3: Write `_sp_plan`
|
||||
|
||||
**Pattern 1: Shard at first block, gather at output** (most common)
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput, SequenceParallelOutput,
|
||||
)
|
||||
|
||||
class StandardTransformer(nn.Module):
|
||||
_sp_plan = {
|
||||
"blocks.0": {
|
||||
"hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
|
||||
},
|
||||
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
}
|
||||
```
|
||||
|
||||
**Pattern 2: Shard RoPE outputs separately**
|
||||
|
||||
```python
|
||||
class TransformerWithRoPE(nn.Module):
|
||||
_sp_plan = {
|
||||
"rope": {
|
||||
0: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),
|
||||
1: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),
|
||||
},
|
||||
"blocks.0": {
|
||||
"hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
|
||||
},
|
||||
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
}
|
||||
```
|
||||
|
||||
**Pattern 3: Dual-stream (shard image, replicate text)**
|
||||
|
||||
```python
|
||||
class DualStreamTransformer(nn.Module):
|
||||
_sp_plan = {
|
||||
"rope_preparer": {
|
||||
2: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True),
|
||||
3: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True),
|
||||
},
|
||||
"transformer_blocks.0": {
|
||||
"hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
|
||||
},
|
||||
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
}
|
||||
```
|
||||
|
||||
### API Reference
|
||||
|
||||
**SequenceParallelInput**:
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `split_dim` | int | Dimension to split (usually 1 for sequence) |
|
||||
| `expected_dims` | int/None | Expected tensor rank for validation |
|
||||
| `split_output` | bool | `False`: shard input params; `True`: shard output tensors |
|
||||
| `auto_pad` | bool | Auto-pad if sequence not divisible by world_size |
|
||||
|
||||
**SequenceParallelOutput**:
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `gather_dim` | int | Dimension to gather (usually 1 for sequence) |
|
||||
| `expected_dims` | int/None | Expected tensor rank for validation |
|
||||
|
||||
**Module naming**:
|
||||
|
||||
| Key | Meaning |
|
||||
|-----|---------|
|
||||
| `"blocks.0"` | First element of ModuleList |
|
||||
| `"blocks.*"` | All elements of ModuleList |
|
||||
| `"rope"` | Named submodule |
|
||||
|
||||
**Dictionary value types**:
|
||||
|
||||
| Key type | split_output | Description |
|
||||
|----------|-------------|-------------|
|
||||
| `"param_name"` (str) | False | Shard input parameter by name |
|
||||
| `0, 1, ...` (int) | True | Shard output tuple by index |
|
||||
|
||||
### Approach 2: Intrusive Modification (Complex Cases)
|
||||
|
||||
For dynamic sharding logic that can't be expressed via `_sp_plan`:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.distributed.sp_sharding import sp_shard, sp_gather
|
||||
|
||||
def forward(self, hidden_states, ...):
|
||||
if self.parallel_config.sequence_parallel_size > 1:
|
||||
hidden_states = sp_shard(hidden_states, dim=1)
|
||||
for block in self.blocks:
|
||||
hidden_states = block(hidden_states)
|
||||
if self.parallel_config.sequence_parallel_size > 1:
|
||||
hidden_states = sp_gather(hidden_states, dim=1)
|
||||
return hidden_states
|
||||
```
|
||||
|
||||
Use intrusive modification as a last resort — `_sp_plan` is preferred for maintainability.
|
||||
|
||||
### UAA Mode (Experimental)
|
||||
|
||||
`ulysses_mode="advanced_uaa"` handles arbitrary sequence lengths and head counts that aren't divisible by `ulysses_degree`. Uses variable all-to-all split sizes and temporary head padding.
|
||||
|
||||
### Combining SP methods
|
||||
|
||||
Ulysses and Ring can be combined: `ulysses_degree × ring_degree = total SP GPUs`.
|
||||
|
||||
```python
|
||||
DiffusionParallelConfig(ulysses_degree=2, ring_degree=2) # 4 GPUs total
|
||||
```
|
||||
|
||||
### Testing SP
|
||||
|
||||
```bash
|
||||
# Offline
|
||||
python text_to_image.py --model Your-model --ulysses-degree 2
|
||||
|
||||
# Online serving
|
||||
vllm serve Your-model --omni --usp 2
|
||||
```
|
||||
|
||||
### Reference implementations
|
||||
|
||||
| Model | Path |
|
||||
|-------|------|
|
||||
| Qwen-Image | `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` |
|
||||
| Wan2.2 | `vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py` |
|
||||
| Z-Image | `vllm_omni/diffusion/models/z_image/z_image_transformer.py` |
|
||||
|
||||
---
|
||||
|
||||
## CFG Parallelism
|
||||
|
||||
Distributes positive/negative Classifier-Free Guidance branches across 2 GPUs.
|
||||
|
||||
### Implementation
|
||||
|
||||
Inherit `CFGParallelMixin` and implement `diffuse()`:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
|
||||
class YourPipeline(nn.Module, CFGParallelMixin):
|
||||
def diffuse(self, latents, timesteps, prompt_embeds, negative_embeds,
|
||||
do_true_cfg, true_cfg_scale, **kwargs):
|
||||
for i, t in enumerate(timesteps):
|
||||
positive_kwargs = {
|
||||
"hidden_states": latents,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"timestep": t,
|
||||
}
|
||||
negative_kwargs = {
|
||||
"hidden_states": latents,
|
||||
"encoder_hidden_states": negative_embeds,
|
||||
"timestep": t,
|
||||
} if do_true_cfg else None
|
||||
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=true_cfg_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
)
|
||||
latents = self.scheduler_step_maybe_with_cfg(
|
||||
noise_pred, t, latents, do_true_cfg
|
||||
)
|
||||
return latents
|
||||
```
|
||||
|
||||
### Customization hooks
|
||||
|
||||
| Method | Override when |
|
||||
|--------|-------------|
|
||||
| `predict_noise()` | Non-standard transformer call (e.g., dual-transformer like Wan2.2) |
|
||||
| `cfg_normalize_function()` | Custom normalization (e.g., LongCat with clamping) |
|
||||
| `combine_cfg_noise()` | Multi-output models (e.g., video + audio: CFG on video, positive-only on audio) |
|
||||
|
||||
**Custom predict_noise** (Wan2.2 — selects active transformer):
|
||||
|
||||
```python
|
||||
def predict_noise(self, current_model=None, **kwargs):
|
||||
if current_model is None:
|
||||
current_model = self.transformer
|
||||
return current_model(**kwargs)[0]
|
||||
```
|
||||
|
||||
**Custom combine_cfg_noise** (multi-output):
|
||||
|
||||
```python
|
||||
def combine_cfg_noise(self, positive_pred, negative_pred, scale, normalize):
|
||||
video_pos, audio_pos = positive_pred
|
||||
video_neg, audio_neg = negative_pred
|
||||
video_combined = super().combine_cfg_noise(video_pos, video_neg, scale, normalize)
|
||||
return (video_combined, audio_pos)
|
||||
```
|
||||
|
||||
### Composite scheduler for multi-output
|
||||
|
||||
When each output has its own schedule:
|
||||
|
||||
```python
|
||||
class VideoAudioScheduler:
|
||||
def __init__(self, video_scheduler, audio_scheduler):
|
||||
self.video_scheduler = video_scheduler
|
||||
self.audio_scheduler = audio_scheduler
|
||||
|
||||
def step(self, noise_pred, t, latents, return_dict=False, generator=None):
|
||||
video_out = self.video_scheduler.step(
|
||||
noise_pred[0], t[0], latents[0], return_dict=False, generator=generator
|
||||
)[0]
|
||||
audio_out = self.audio_scheduler.step(
|
||||
noise_pred[1], t[1], latents[1], return_dict=False, generator=generator
|
||||
)[0]
|
||||
return ((video_out, audio_out),)
|
||||
```
|
||||
|
||||
### Testing CFG Parallel
|
||||
|
||||
```bash
|
||||
python text_to_image.py --model Your-model \
|
||||
--cfg-parallel-size 2 --cfg-scale 4.0 \
|
||||
--negative-prompt "ugly, unclear"
|
||||
```
|
||||
|
||||
**Constraint**: `guidance_scale > 1.0` and negative prompt must be provided.
|
||||
|
||||
### Reference implementations
|
||||
|
||||
| Model | Path |
|
||||
|-------|------|
|
||||
| Qwen-Image | `vllm_omni/diffusion/models/qwen_image/cfg_parallel.py` |
|
||||
| Wan2.2 | `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py` |
|
||||
| Mixin base | `vllm_omni/diffusion/distributed/cfg_parallel.py` |
|
||||
|
||||
---
|
||||
|
||||
## HSDP (Hybrid Sharded Data Parallel)
|
||||
|
||||
Shards model weights across GPUs using PyTorch FSDP2. Reduces per-GPU VRAM without changing computation.
|
||||
|
||||
### Implementation
|
||||
|
||||
Add `_hsdp_shard_conditions` to the transformer class:
|
||||
|
||||
```python
|
||||
class YourTransformer(nn.Module):
|
||||
@staticmethod
|
||||
def _is_transformer_block(name: str, module) -> bool:
|
||||
return "blocks" in name and name.split(".")[-1].isdigit()
|
||||
|
||||
_hsdp_shard_conditions = [_is_transformer_block]
|
||||
```
|
||||
|
||||
For MoE models, add additional conditions:
|
||||
|
||||
```python
|
||||
class MoETransformer(nn.Module):
|
||||
@staticmethod
|
||||
def _is_transformer_block(name, module):
|
||||
return "blocks" in name and name.split(".")[-1].isdigit()
|
||||
|
||||
@staticmethod
|
||||
def _is_moe_expert(name, module):
|
||||
return "experts" in name and name.split(".")[-1].isdigit()
|
||||
|
||||
_hsdp_shard_conditions = [_is_transformer_block, _is_moe_expert]
|
||||
```
|
||||
|
||||
A module is sharded if **any** condition returns `True`.
|
||||
|
||||
### Constraints
|
||||
|
||||
- Cannot combine with Tensor Parallelism
|
||||
- For standalone HSDP (no other parallelism), `hsdp_shard_size` must be specified explicitly
|
||||
- Can combine with SP: HSDP reduces memory while SP distributes sequence
|
||||
|
||||
### Testing HSDP
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
|
||||
parallel_config = DiffusionParallelConfig(use_hsdp=True, hsdp_shard_size=8)
|
||||
omni = Omni(model="your-model", parallel_config=parallel_config)
|
||||
```
|
||||
|
||||
Or CLI:
|
||||
|
||||
```bash
|
||||
vllm serve Your-model --omni --use-hsdp
|
||||
```
|
||||
|
||||
**Verify**: logs show "HSDP Inference: replicate_size=..., shard_size=..." and "Sharded N modules + root". Check VRAM reduction.
|
||||
|
||||
### Reference implementations
|
||||
|
||||
| Model | Path |
|
||||
|-------|------|
|
||||
| Wan2.2 | `vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py` |
|
||||
| HSDP Core | `vllm_omni/diffusion/distributed/hsdp.py` |
|
||||
|
||||
---
|
||||
|
||||
## VAE Patch Parallelism
|
||||
|
||||
Shards VAE decode spatially across ranks using tiling:
|
||||
|
||||
```bash
|
||||
python text_to_image.py --model Your-model --vae-patch-parallel-size 4
|
||||
```
|
||||
|
||||
Auto-enables `--vae-use-tiling`. Uses `DistributedAutoencoderKLWan` or similar distributed VAE. Set `vae_patch_parallel_size` in `DiffusionParallelConfig`.
|
||||
|
||||
---
|
||||
|
||||
## Combining Parallelism Methods
|
||||
|
||||
Common multi-GPU recipes:
|
||||
|
||||
```bash
|
||||
# 4 GPUs: CFG (2) × Ulysses (2)
|
||||
python text_to_image.py --model Qwen/Qwen-Image \
|
||||
--cfg-parallel-size 2 --ulysses-degree 2
|
||||
|
||||
# 8 GPUs: Ulysses (4) × Ring (2) + VAE patch (8)
|
||||
python text_to_video.py --model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
|
||||
--ulysses-degree 4 --ring-degree 2 --vae-patch-parallel-size 8
|
||||
|
||||
# 2 GPUs: HSDP + Ulysses (cannot combine HSDP with TP)
|
||||
vllm serve Your-model --omni --use-hsdp --usp 2
|
||||
```
|
||||
|
||||
## Discovering Parallelism Support
|
||||
|
||||
Check which parallelism methods a model supports:
|
||||
|
||||
| Check | How |
|
||||
|-------|-----|
|
||||
| **Ulysses / Ring SP** | Transformer defines `_sp_plan`. Search: `grep -r '_sp_plan' vllm_omni/diffusion/models/` |
|
||||
| **CFG Parallel** | Pipeline inherits `CFGParallelMixin`. Search: `grep -r 'CFGParallelMixin' vllm_omni/diffusion/models/` |
|
||||
| **TP** | Uses `ColumnParallelLinear` / `QKVParallelLinear`. Search: `grep -r 'ParallelLinear\|QKVParallel' vllm_omni/diffusion/models/<model>/` |
|
||||
| **HSDP** | Transformer defines `_hsdp_shard_conditions`. Search: `grep -r '_hsdp_shard_conditions' vllm_omni/diffusion/models/` |
|
||||
|
||||
The canonical per-model support overview/table is in `docs/user_guide/diffusion/parallelism/overview.md`.
|
||||
@@ -1,239 +0,0 @@
|
||||
# Transformer Adaptation Reference
|
||||
|
||||
## Adapting a Diffusers Transformer to vLLM-Omni
|
||||
|
||||
### Step-by-step Checklist
|
||||
|
||||
1. Copy the transformer class from diffusers source
|
||||
2. Remove all mixin classes — inherit only from `nn.Module`
|
||||
3. Replace attention dispatch with `vllm_omni.diffusion.attention.layer.Attention`
|
||||
4. Replace logger with `vllm.logger.init_logger`
|
||||
5. Add `od_config: OmniDiffusionConfig | None = None` to `__init__`
|
||||
6. Remove training-only code (gradient checkpointing, dropout)
|
||||
7. Add `load_weights()` method for weight loading from safetensors
|
||||
8. Add class-level attributes for acceleration features
|
||||
|
||||
### Mixin Removal
|
||||
|
||||
Remove these diffusers mixins (and their imports):
|
||||
|
||||
```python
|
||||
# Remove all of these:
|
||||
from diffusers.models.modeling_utils import ModelMixin
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.models.attention_processor import AttentionModuleMixin
|
||||
from diffusers.loaders import PeftAdapterMixin, FromOriginalModelMixin
|
||||
|
||||
# Replace:
|
||||
class MyTransformer(ModelMixin, ConfigMixin, AttentionModuleMixin):
|
||||
# With:
|
||||
class MyTransformer(nn.Module):
|
||||
```
|
||||
|
||||
Also remove `@register_to_config` decorators from `__init__`.
|
||||
|
||||
### Attention Replacement
|
||||
|
||||
The vLLM-Omni `Attention` layer wraps backend selection (FlashAttention, SDPA, SageAttn, etc.) and supports sequence parallelism hooks.
|
||||
|
||||
**QKV tensor shape must be `[batch, seq_len, num_heads, head_dim]`.**
|
||||
|
||||
#### Self-Attention Pattern
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
|
||||
class SelfAttentionBlock(nn.Module):
|
||||
def __init__(self, dim, num_heads):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
|
||||
self.to_q = nn.Linear(dim, dim)
|
||||
self.to_k = nn.Linear(dim, dim)
|
||||
self.to_v = nn.Linear(dim, dim)
|
||||
self.to_out = nn.Linear(dim, dim)
|
||||
|
||||
self.attn = Attention(
|
||||
num_heads=num_heads,
|
||||
head_size=self.head_dim,
|
||||
softmax_scale=1.0 / (self.head_dim ** 0.5),
|
||||
causal=False,
|
||||
num_kv_heads=num_heads,
|
||||
role="self",
|
||||
)
|
||||
|
||||
def forward(self, x, attn_mask=None):
|
||||
B, S, _ = x.shape
|
||||
q = self.to_q(x).view(B, S, self.num_heads, self.head_dim)
|
||||
k = self.to_k(x).view(B, S, self.num_heads, self.head_dim)
|
||||
v = self.to_v(x).view(B, S, self.num_heads, self.head_dim)
|
||||
|
||||
attn_metadata = AttentionMetadata(attn_mask=attn_mask)
|
||||
out = self.attn(q, k, v, attn_metadata=attn_metadata)
|
||||
out = out.reshape(B, S, -1)
|
||||
return self.to_out(out)
|
||||
```
|
||||
|
||||
**Declaring the attention role.** Always pass a `role` string to `Attention(...)`. It is the lookup key users target with `--diffusion-attention-config.per_role.<role>.backend` to swap kernels per site without touching model code.
|
||||
|
||||
| Convention | When to use |
|
||||
|---|---|
|
||||
| `role="self"` | Q/K/V come from the same hidden state |
|
||||
| `role="cross"` | K/V come from a separate `encoder_hidden_states`; pair with `skip_sequence_parallel=True` if K/V is replicated across SP ranks |
|
||||
|
||||
For multi-modal sites that don't fit `self` / `cross`, use a dot-namespaced role and provide `role_category` so user config can fall back to the generic category:
|
||||
|
||||
```python
|
||||
self.audio_to_video_attn = Attention(
|
||||
num_heads=num_heads, head_size=self.head_dim,
|
||||
softmax_scale=1.0 / (self.head_dim ** 0.5), causal=False,
|
||||
role="mymodel.audio_to_video",
|
||||
role_category="cross",
|
||||
)
|
||||
```
|
||||
|
||||
#### Fused QKV with TP (Advanced)
|
||||
|
||||
For tensor parallelism, use vLLM's parallel linear layers:
|
||||
|
||||
```python
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear, RowParallelLinear
|
||||
)
|
||||
|
||||
class TPSelfAttention(nn.Module):
|
||||
def __init__(self, dim, num_heads):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
|
||||
self.to_qkv = QKVParallelLinear(
|
||||
hidden_size=dim,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=num_heads,
|
||||
total_num_kv_heads=num_heads,
|
||||
)
|
||||
self.to_out = RowParallelLinear(dim, dim)
|
||||
|
||||
self.attn = Attention(
|
||||
num_heads=num_heads,
|
||||
head_size=self.head_dim,
|
||||
softmax_scale=1.0 / (self.head_dim ** 0.5),
|
||||
causal=False,
|
||||
num_kv_heads=num_heads,
|
||||
role="self",
|
||||
)
|
||||
```
|
||||
|
||||
### Logger Replacement
|
||||
|
||||
```python
|
||||
# Replace:
|
||||
from diffusers.utils import logging
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
# With:
|
||||
from vllm.logger import init_logger
|
||||
logger = init_logger(__name__)
|
||||
```
|
||||
|
||||
### Custom Layers from vLLM-Omni
|
||||
|
||||
Available utility layers:
|
||||
|
||||
```python
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
|
||||
```
|
||||
|
||||
### Config Support
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
class MyTransformer(nn.Module):
|
||||
def __init__(self, *, od_config=None, num_layers=28, hidden_size=3072, **kwargs):
|
||||
super().__init__()
|
||||
self.od_config = od_config
|
||||
self.parallel_config = od_config.parallel_config if od_config else None
|
||||
# ... build layers
|
||||
```
|
||||
|
||||
The transformer config values come from `model_index.json` → `config.json` in the transformer subfolder. The pipeline uses `get_transformer_config_kwargs(od_config.tf_model_config, TransformerClass)` to filter config keys to match the `__init__` signature.
|
||||
|
||||
### Weight Loading
|
||||
|
||||
The `load_weights` method receives an iterable of `(name, tensor)` from safetensors files, with the prefix (e.g., `"transformer."`) already stripped by the loader.
|
||||
|
||||
```python
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
class MyTransformer(nn.Module):
|
||||
def load_weights(self, weights):
|
||||
params = dict(self.named_parameters())
|
||||
loaded = set()
|
||||
for name, tensor in weights:
|
||||
# Optional: remap names from diffusers to vllm-omni naming
|
||||
# e.g., "ff.net.0.proj" -> "ff.net_0.proj"
|
||||
|
||||
if name in params:
|
||||
param = params[name]
|
||||
if hasattr(param, "weight_loader"):
|
||||
param.weight_loader(param, tensor)
|
||||
else:
|
||||
default_weight_loader(param, tensor)
|
||||
loaded.add(name)
|
||||
return loaded
|
||||
```
|
||||
|
||||
#### QKV Fusion in load_weights
|
||||
|
||||
If you fused separate Q/K/V into a `QKVParallelLinear`, you need to map diffusers' separate weight names:
|
||||
|
||||
```python
|
||||
stacked_params_mapping = [
|
||||
("to_qkv", "to_q", "q"),
|
||||
("to_qkv", "to_k", "k"),
|
||||
("to_qkv", "to_v", "v"),
|
||||
]
|
||||
|
||||
def load_weights(self, weights):
|
||||
params = dict(self.named_parameters())
|
||||
loaded = set()
|
||||
for name, tensor in weights:
|
||||
for fused_name, orig_name, shard_id in stacked_params_mapping:
|
||||
if orig_name in name:
|
||||
name = name.replace(orig_name, fused_name)
|
||||
param = params[name]
|
||||
param.weight_loader(param, tensor, shard_id)
|
||||
loaded.add(name)
|
||||
break
|
||||
else:
|
||||
# Normal loading
|
||||
...
|
||||
return loaded
|
||||
```
|
||||
|
||||
### Class-Level Attributes for Features
|
||||
|
||||
|
||||
```python
|
||||
class MyTransformer(nn.Module):
|
||||
# torch.compile: list block class names that repeat and can be compiled
|
||||
_repeated_blocks = ["MyTransformerBlock"]
|
||||
|
||||
# CPU offload: attribute name of the nn.ModuleList containing blocks
|
||||
_layerwise_offload_blocks_attrs = ["blocks"]
|
||||
|
||||
# LoRA: mapping of fused param names to original param names
|
||||
packed_modules_mapping = {"to_qkv": ["to_q", "to_k", "to_v"]}
|
||||
|
||||
# Sequence parallelism plan (advanced — add after basic impl works)
|
||||
_sp_plan = {
|
||||
"blocks.0": SequenceParallelInput(split_dim=1),
|
||||
"proj_out": SequenceParallelOutput(gather_dim=1),
|
||||
}
|
||||
```
|
||||
@@ -1,178 +0,0 @@
|
||||
# Troubleshooting Reference
|
||||
|
||||
## Common Errors When Adding a Diffusion Model
|
||||
|
||||
### ImportError / ModuleNotFoundError
|
||||
|
||||
**Cause**: Missing or incorrect registration.
|
||||
|
||||
**Fix checklist**:
|
||||
1. Model registered in `vllm_omni/diffusion/registry.py` `_DIFFUSION_MODELS` dict
|
||||
2. `__init__.py` exports the pipeline class
|
||||
3. Pipeline file exists at the correct path: `vllm_omni/diffusion/models/{folder}/{file}.py`
|
||||
4. Class name in registry matches the actual class name in the file
|
||||
|
||||
### Shape Mismatch in Attention
|
||||
|
||||
**Symptom**: `RuntimeError: shape mismatch` or `expected 4D tensor`
|
||||
|
||||
**Cause**: QKV tensors not reshaped to `[batch, seq_len, num_heads, head_dim]`.
|
||||
|
||||
**Fix**: Before calling `self.attn(q, k, v, ...)`, ensure:
|
||||
```python
|
||||
q = q.view(batch, seq_len, self.num_heads, self.head_dim)
|
||||
k = k.view(batch, kv_seq_len, self.num_kv_heads, self.head_dim)
|
||||
v = v.view(batch, kv_seq_len, self.num_kv_heads, self.head_dim)
|
||||
```
|
||||
|
||||
After attention, reshape back:
|
||||
```python
|
||||
out = out.reshape(batch, seq_len, -1)
|
||||
```
|
||||
|
||||
### Weight Loading Failures
|
||||
|
||||
**Symptom**: `RuntimeError: size mismatch for parameter ...` or missing keys
|
||||
|
||||
**Debugging**:
|
||||
1. Print diffusers weight names: `safetensors.safe_open(path, "pt").keys()`
|
||||
2. Print model parameter names: `dict(model.named_parameters()).keys()`
|
||||
3. Compare and add name remappings in `load_weights()`
|
||||
|
||||
**Common remappings needed**:
|
||||
- `ff.net.0.proj` → `ff.net_0.proj` (PyTorch Sequential indexing)
|
||||
- `.to_out.0.` → `.to_out.` (Sequential unwrapping)
|
||||
- `scale_shift_table` → moved to a wrapper module
|
||||
|
||||
### Black/Blank/Noisy Output
|
||||
|
||||
**Possible causes**:
|
||||
1. **Wrong latent normalization**: Check VAE expects latents scaled by `vae.config.scaling_factor`
|
||||
2. **Wrong scheduler**: Using the wrong scheduler class or wrong `flow_shift`
|
||||
3. **Missing CFG**: Some models require `guidance_scale > 1.0` with negative prompt
|
||||
4. **Wrong timestep format**: Some schedulers expect float, others expect int/long
|
||||
5. **Missing post-processing**: Raw VAE output may need denormalization
|
||||
|
||||
**Quick test**: Run with diffusers directly using the same seed and compare latents at each step.
|
||||
|
||||
### OOM (Out of Memory)
|
||||
|
||||
**Solutions** (in order of preference):
|
||||
1. `--enforce-eager` to disable torch.compile (saves compile memory)
|
||||
2. `--enable-cpu-offload` for model-level offload
|
||||
3. `--enable-layerwise-offload` for block-level offload (better for large models)
|
||||
4. `--vae-use-slicing --vae-use-tiling` for VAE memory reduction
|
||||
5. Reduce resolution: `--height 480 --width 832`
|
||||
6. Use TP: `--tensor-parallel-size 2`
|
||||
|
||||
### Different Output vs Diffusers Reference
|
||||
|
||||
**Common causes**:
|
||||
1. **Attention backend difference**: FlashAttention vs SDPA may produce slightly different results. Force SDPA via `--diffusion-attention-backend TORCH_SDPA` (or `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA`) to match diffusers; for finer-grained per-role overrides see `--diffusion-attention-config.per_role.<role>.backend`
|
||||
2. **Float precision**: vLLM-Omni may use bfloat16 where diffusers uses float32 for some operations
|
||||
3. **Missing normalization**: Check all LayerNorm/RMSNorm are preserved
|
||||
4. **Scheduler rounding**: Some schedulers have numerical sensitivity
|
||||
|
||||
### Tensor Parallel Errors
|
||||
|
||||
**Symptom**: `AssertionError: not divisible` or incorrect output with TP>1
|
||||
|
||||
**Fix**:
|
||||
1. Verify `num_heads % tp_size == 0` and `num_kv_heads % tp_size == 0`
|
||||
2. Ensure `ColumnParallelLinear` / `RowParallelLinear` are used correctly
|
||||
3. Check that norms between parallel layers use distributed norm if needed
|
||||
4. Verify `load_weights` handles TP sharding for norm weights
|
||||
5. Use `self.to_qkv.num_heads` (local heads per GPU) for QKV split sizes, not total heads
|
||||
|
||||
**Missing `input_is_parallel=True`**:
|
||||
|
||||
`RowParallelLinear` expects sharded input from `ColumnParallelLinear`:
|
||||
```python
|
||||
self.w1 = ColumnParallelLinear(dim, hidden_dim, return_bias=False)
|
||||
self.w2 = RowParallelLinear(hidden_dim, dim, input_is_parallel=True, return_bias=False)
|
||||
```
|
||||
|
||||
### Sequence Parallel Errors
|
||||
|
||||
**Symptom**: Incorrect output or crashes with `--ulysses-degree N` or `--usp N`
|
||||
|
||||
**Possible causes**:
|
||||
1. **Inline operations between shard/gather points**: `torch.cat()`, `pad_sequence()` etc. not at `nn.Module` boundaries. Fix: extract into submodule.
|
||||
2. **Wrong `split_dim`**: Check the tensor shape at the shard point. Sequence dimension is typically `dim=1` for `[B, S, D]` tensors.
|
||||
3. **RoPE not sharded**: If RoPE is computed separately, add it to `_sp_plan` with `split_output=True`.
|
||||
4. **Sequence not divisible by SP degree**: Use `auto_pad=True` in `SequenceParallelInput` or switch to `ulysses_mode="advanced_uaa"`.
|
||||
|
||||
**Debugging**: Add `expected_dims=N` to `SequenceParallelInput`/`Output` for shape validation at runtime.
|
||||
|
||||
### CFG Parallel Errors
|
||||
|
||||
**Symptom**: CFG parallel not activating, no speedup
|
||||
|
||||
**Fix checklist**:
|
||||
1. Pipeline inherits `CFGParallelMixin`
|
||||
2. `guidance_scale > 1.0`
|
||||
3. Negative prompt provided (even if empty string)
|
||||
4. `--cfg-parallel-size 2` specified
|
||||
5. `diffuse()` method calls `predict_noise_maybe_with_cfg()` and `scheduler_step_maybe_with_cfg()`
|
||||
|
||||
**Symptom**: Different output with CFG parallel vs sequential
|
||||
|
||||
**Possible cause**: Non-deterministic scheduler. Fix: pass `generator=torch.Generator(device).manual_seed(seed)` to `scheduler_step_maybe_with_cfg()`.
|
||||
|
||||
### HSDP Errors
|
||||
|
||||
**Symptom**: HSDP not activating or errors during weight loading
|
||||
|
||||
**Fix checklist**:
|
||||
1. Transformer defines `_hsdp_shard_conditions` class attribute
|
||||
2. Shard condition functions return `True` for correct modules (test with `model.named_modules()`)
|
||||
3. Not combining with TP (HSDP and TP are incompatible)
|
||||
4. For standalone HSDP, `hsdp_shard_size` is specified explicitly
|
||||
|
||||
**Verify**: Check logs for "HSDP Inference: replicate_size=..., shard_size=..." and "Sharded N modules + root".
|
||||
|
||||
### Cache-DiT Not Applied
|
||||
|
||||
**Symptom**: No speedup, no cache-related log messages
|
||||
|
||||
**Fix checklist**:
|
||||
1. Model not in `_NO_CACHE_ACCELERATION` in `registry.py`
|
||||
2. Pipeline class name matches `CUSTOM_DIT_ENABLERS` key (if using custom enabler)
|
||||
3. `cache_backend="cache_dit"` specified
|
||||
4. Check logs for "Cache-dit enabled successfully on xxx"
|
||||
|
||||
**Verify pipeline name**: `print(pipeline.__class__.__name__)` — must match registry key.
|
||||
|
||||
### Cache-DiT Quality Degradation
|
||||
|
||||
**Symptom**: Artifacts or lower quality with cache-dit
|
||||
|
||||
**Fix**: Reduce aggressiveness:
|
||||
```python
|
||||
cache_config={
|
||||
"residual_diff_threshold": 0.12, # Lower from 0.24
|
||||
"max_warmup_steps": 6, # Increase from 4
|
||||
"max_continuous_cached_steps": 2, # Reduce if higher
|
||||
}
|
||||
```
|
||||
|
||||
If quality is still poor, the model may need a custom enabler with per-block-list `ParamsModifier` tuning.
|
||||
|
||||
### Model Not Detected / Wrong Pipeline Class
|
||||
|
||||
**Symptom**: `ValueError: Model class ... not found in diffusion model registry`
|
||||
|
||||
**Cause**: The model's `model_index.json` has a `_class_name` for the pipeline that doesn't match registry keys.
|
||||
|
||||
**Fix**: The registry key must match the diffusers pipeline class name from `model_index.json`. If using a different name, map it in the registry:
|
||||
```python
|
||||
"DiffusersPipelineClassName": ("your_folder", "your_file", "YourVllmClassName"),
|
||||
```
|
||||
|
||||
## Debugging Workflow
|
||||
|
||||
1. **Add verbose logging**: Use `logger.info()` to print tensor shapes at each stage
|
||||
2. **Compare step-by-step**: Run diffusers and vllm-omni side by side, comparing tensors after each major operation
|
||||
3. **Use small configs**: Reduce `num_inference_steps=2`, small resolution for fast iteration
|
||||
4. **Test transformer isolation**: Feed the same input to both diffusers and vllm-omni transformers, compare outputs
|
||||
5. **Binary search for bugs**: Comment out blocks/layers to isolate where divergence starts
|
||||
@@ -1,584 +0,0 @@
|
||||
---
|
||||
name: add-tts-model
|
||||
description: "Integrate a new text-to-speech model into vLLM-Omni from HuggingFace reference implementation through production-ready serving with streaming and CUDA graph acceleration. Use when adding a new TTS model, wiring stage separation for speech synthesis, enabling online voice generation serving, debugging TTS integration behavior, or building audio output pipelines."
|
||||
---
|
||||
|
||||
# TTS Model Integration Workflow
|
||||
|
||||
## Overview
|
||||
|
||||
```
|
||||
HF Reference -> Stage Separation -> Online Serving -> Async Chunk -> CUDA Graph -> Pre-commit/DCO
|
||||
(Phase 1) (Phase 2) (Phase 3) (Phase 4) (Phase 5) (Phase 6)
|
||||
```
|
||||
|
||||
Three architecture patterns are supported:
|
||||
|
||||
- **Two-stage pipeline** (e.g. Qwen3-TTS, Fish Speech, CosyVoice3): AR
|
||||
code-predictor → audio decoder, connected via async_chunk for low-latency
|
||||
streaming. Use this for maximum performance.
|
||||
- **Single-stage AR via generator** (e.g. MOSS-TTS-Nano): entire model runs
|
||||
inside one AR worker, streaming audio chunks through a per-request
|
||||
`inference_stream()` generator. Use this when the upstream model bundles AR
|
||||
+ codec inseparably. See [references/single-stage-ar.md](references/single-stage-ar.md).
|
||||
- **Single-stage, vLLM-native base LM + side computation** (e.g. VoxCPM2):
|
||||
the base language model runs under vLLM's PagedAttention as a normal AR
|
||||
model; diffusion / VAE / side computations run outside vLLM and are
|
||||
attached via the runner post-processing hook. This is a distinct pattern
|
||||
from the generator approach above — do not confuse the two.
|
||||
|
||||
The single-stage variants skip Phase 4 (async_chunk) but Phase 5 (CUDA graph)
|
||||
is still encouraged for the inner AR loop.
|
||||
|
||||
## Cross-Cutting Invariants
|
||||
|
||||
These rules apply to every TTS model regardless of architecture (AR vs AR+diffusion, single-stage vs two-stage, codec-based vs VAE-based). They surface repeatedly across PRs — check them at the end of every phase.
|
||||
|
||||
### I1. Streaming output contract
|
||||
|
||||
Pick exactly one per-step semantics for `forward()` and document it in the docstring:
|
||||
|
||||
- **Delta**: yield only new audio samples produced this step. Preferred — linear cost, low memory.
|
||||
- **Cumulative**: re-decode from step 0 every call. O(N²); only acceptable if the codec has no streaming decode path.
|
||||
|
||||
If you choose **delta**, verify the full emit→consolidate→consume chain:
|
||||
|
||||
1. `forward()` returns `{"model_outputs": <new_chunk_only>, ...}`
|
||||
2. `_consolidate_multimodal_tensors()` in `vllm_omni/engine/output_processor.py` concatenates the audio key into one tensor at finish. If it skips the key (`continue`), offline consumers receive only the final chunk. See `output_processor.py` for the concrete list of handled modality keys.
|
||||
3. Streaming consumers (SSE, Gradio) receive per-step deltas; offline consumers (`engine.generate()`) receive a single concatenated tensor.
|
||||
|
||||
Cumulative-vs-delta mismatch is the most common silent bug — offline RTF benchmarks pass, but users hear replays or truncation.
|
||||
|
||||
### I2. Multimodal output consumer hygiene
|
||||
|
||||
`outputs[0].outputs[0].multimodal_output[<key>]` can be any of `Tensor`, `list[Tensor]` (pre-consolidation snapshot), `np.ndarray`, or scalar. When writing tests, examples, and benchmarks:
|
||||
|
||||
- **Never** use `dict.get("a") or dict.get("b")` on tensor values — Python evaluates the tensor's boolean, raising `RuntimeError: Boolean value of Tensor with more than one value is ambiguous`. Use explicit `if x is None` chains.
|
||||
- Always defensively handle the list form: `if isinstance(x, list): x = torch.cat([t.reshape(-1) for t in x], dim=0)`.
|
||||
- Assert `shape` / `dtype` / `duration` explicitly; do not rely on truthiness for presence checks.
|
||||
|
||||
### I3. Hot-loop GPU discipline
|
||||
|
||||
Inside any per-step model loop (AR decode, diffusion solver, CFM Euler, vocoder block loop):
|
||||
|
||||
- No `tensor.item()`, `.cpu()`, or `.tolist()` — each triggers a GPU→CPU sync; at 10 steps × 60 frames × 4 ops that is 2400 syncs per request.
|
||||
- Prefer `dst.copy_(src)` over `dst.fill_(src.item())` when writing a scalar tensor into a buffer.
|
||||
- Prefer `torch.compile(Model.forward, fullgraph=False)` on the whole forward over per-submodule compile — fewer dispatch boundaries, larger fusion regions. Measure before choosing granularity.
|
||||
- No Python-side control flow that depends on tensor values; use `torch.where` / masking instead.
|
||||
|
||||
Profile first, optimize second. See the profiling docs / project memory for the trace-analysis workflow.
|
||||
|
||||
### I4. Validation pyramid
|
||||
|
||||
Offline RTF alone is necessary but not sufficient. Every new TTS model must pass all three:
|
||||
|
||||
| Layer | Catches | Tool |
|
||||
|-------|---------|------|
|
||||
| Offline RTF / duration check | Throughput regressions, missing audio, wrong sample rate | `end2end.py`, pytest e2e |
|
||||
| Browser streaming playback | Delta/cumulative bugs, chunk boundary glitches, TTFP regressions | Gradio demo over `/v1/audio/speech?stream=true` |
|
||||
| Concurrent requests | Per-request state leaks, codec window round-robin gaps | `max_num_seqs>1` smoke test with 4+ parallel prompts |
|
||||
|
||||
Declaring a model "done" without all three has shipped regressions more than once.
|
||||
|
||||
### I5. Per-request state is owned by the request, not the model
|
||||
|
||||
If the model caches *anything* across `forward()` calls (streaming generators, codec buffers, sliding-window pads, CUDA graph state), key it by request ID:
|
||||
|
||||
```python
|
||||
self._state: dict[str, YourState] = {} # request_key → state
|
||||
# fetch: request_key = str(info.get("_omni_req_id", "0"))
|
||||
# free on finish: del self._state[request_key]
|
||||
```
|
||||
|
||||
A shared buffer silently corrupts audio across concurrent requests — the symptom is crosstalk or truncation only under load.
|
||||
|
||||
## Phase 1: HuggingFace Reference
|
||||
|
||||
**Goal**: Understand the reference implementation and verify it produces correct audio.
|
||||
|
||||
### Steps
|
||||
|
||||
1. **Run the reference model** end-to-end using the official HuggingFace / GitHub code
|
||||
2. **Document the architecture**:
|
||||
- What are the sub-models? (AR decoder, codec decoder, vocoder, etc.)
|
||||
- What is the token vocabulary? (semantic codes, RVQ codebooks, special tokens)
|
||||
- What is the output format? (sample rate, channels, codec type)
|
||||
3. **Capture reference outputs** for comparison during integration
|
||||
4. **Identify the config structure**: `config.json` fields, `model_type`, sub-model configs
|
||||
|
||||
### Key Questions
|
||||
|
||||
- How many codebooks? What are the codebook sizes?
|
||||
- What special tokens exist? (`<|voice|>`, `<|audio_start|>`, `<|im_end|>`, etc.)
|
||||
- What is the token-to-ID mapping for codec codes?
|
||||
- What is the hop length / frame rate of the codec?
|
||||
- Does the model support voice cloning? How? (reference audio encoding, speaker embeddings, etc.)
|
||||
|
||||
### Deliverables
|
||||
|
||||
- Working reference script that produces audio
|
||||
- Architecture diagram / notes
|
||||
- Token vocabulary mapping
|
||||
- Reference audio samples for regression testing
|
||||
|
||||
## Phase 2: Stage Separation (Offline Inference)
|
||||
|
||||
**Goal**: Split the model into vLLM-Omni stages and get offline inference working.
|
||||
|
||||
### Steps
|
||||
|
||||
1. **Register the model** in `vllm_omni/model_executor/models/registry.py`
|
||||
2. **Create config classes** (`configuration_<model>.py`) with `model_type` registration
|
||||
3. **Implement Stage 0** (AR model):
|
||||
- Subclass appropriate base (e.g., wrap Qwen3 decoder layers)
|
||||
- Implement `forward()` for autoregressive token generation
|
||||
- Handle special token logic (start/stop tokens, codec token mapping)
|
||||
- If dual-AR (like Fish Speech), implement Fast AR as a nested module
|
||||
4. **Implement Stage 1** (Decoder):
|
||||
- Load codec weights (may need lazy loading from separate checkpoint)
|
||||
- Implement `forward()`: codec codes -> audio waveform
|
||||
- Return `OmniOutput` with `multimodal_outputs`
|
||||
5. **Define and register the pipeline topology** in
|
||||
`vllm_omni/model_executor/models/<model>/pipeline.py`
|
||||
6. **Create the deploy YAML** under `vllm_omni/deploy/` for placement,
|
||||
memory sizing, connectors, and runtime overrides
|
||||
7. **Create stage input processor** for prompt building and inter-stage handoff
|
||||
8. **Write end2end.py** test script
|
||||
|
||||
### Critical Parameters to Get Right
|
||||
|
||||
| Parameter | Impact if Wrong |
|
||||
|-----------|----------------|
|
||||
| Hop length | Audio duration wrong, streaming noise |
|
||||
| Token ID mapping | Garbage codes -> noise output |
|
||||
| Codebook count/size | Shape mismatch crashes |
|
||||
| Stop token | Generation never stops or stops too early |
|
||||
| dtype / autocast | Numerical issues, silent quality degradation |
|
||||
| Repetition penalty | Must match reference (often 1.0 for TTS) |
|
||||
|
||||
### Debugging Priority (from experience)
|
||||
|
||||
When audio output is wrong, check in this order:
|
||||
|
||||
1. **RoPE / attention**: Are position encodings correct? Is the attention mask right?
|
||||
2. **Normalization**: RMSNorm epsilon, layer norm placement (pre vs post)
|
||||
3. **Hop length**: Product of all upsample rates in the codec decoder
|
||||
4. **Token mapping**: Are codec IDs correctly offset from the vocabulary base?
|
||||
5. **Sampling parameters**: Temperature, top_k, top_p, repetition_penalty
|
||||
6. **Tensor layout**: Codebook-major vs frame-major ordering
|
||||
7. **dtype**: Float32 for codec decoders (autocast can corrupt audio)
|
||||
|
||||
### Streaming Correctness Rules (single-stage and two-stage)
|
||||
|
||||
These bugs appear in almost every new TTS PR. Check all before the first push. See also the cross-cutting invariants I1 (output contract) and I5 (per-request state) above — the rules below are the Phase 2-specific instances of those invariants:
|
||||
|
||||
- **Accumulate codes across AR steps** — each `forward()` appends new codes; do not reset between steps or audio will be truncated (fish speech: `fix: accumulate audio_codes across steps`)
|
||||
- **Emit delta audio, not full waveform** — in streaming mode yield only the new chunk per step, not the re-decoded full waveform from step 0 (fish speech: `fix: emit delta audio not full waveform`)
|
||||
- **All return paths must emit `model_outputs`** — if any early-return branch skips setting `model_outputs`, the serving layer silently drops that step's audio (fish speech: `fix: ensure ALL return paths emit model_outputs`)
|
||||
- **Per-request state isolation** — for batched concurrent requests, key all state by request ID; a shared buffer corrupts audio across requests (fish speech: `fix: per-request vocode + delta emission`)
|
||||
- **Codec tensor device** — move codec codes to the codec decoder's device before calling decode; mismatches cause silent CPU fallback or crashes (fish speech: `fix: use model device for CUDA stream`)
|
||||
- **AR stage `max_num_seqs`** — set to at least 4 in production deploy configs; for single-stage models this is the only stage. For two-stage models, Stage 0 (AR) needs `max_num_seqs ≥ 4` to pipeline concurrent requests; Stage 1 (codec decoder) is model-specific and may intentionally use `max_num_seqs: 1`. Defaulting the AR stage to 1 causes audio gaps under concurrency because the codec window round-robins across requests (RFC #2568)
|
||||
|
||||
### Optional Dependency Handling
|
||||
|
||||
Patch optional dependencies (`torchaudio` / `torchcodec` / `soundfile`) at
|
||||
the top of `load_weights()`, not at module import. Failures to do so cause
|
||||
cryptic errors only on environments missing the optional package — after
|
||||
the model is already deployed. See
|
||||
[references/optional-deps.md](references/optional-deps.md) for the full
|
||||
pattern, signature constraints, and MOSS-TTS-Nano reference.
|
||||
|
||||
### Single-Stage AR Pattern (alternative to two-stage)
|
||||
|
||||
When the upstream model cannot be cleanly split into an AR stage and a
|
||||
separate decoder, run the full pipeline inside a single AR worker and
|
||||
stream audio through a per-request `inference_stream()` generator keyed by
|
||||
`_omni_req_id`. Define one `StagePipelineConfig` with
|
||||
`execution_type=StageExecutionType.LLM_AR`, `engine_output_type="audio"`,
|
||||
`final_output=True`, and `owns_tokenizer=True`. Set `async_chunk: false` in
|
||||
the deploy YAML. Only extract params from
|
||||
`additional_information` that you actually forward, or pre-commit fails
|
||||
`ruff F841`.
|
||||
|
||||
Full walkthrough with the complete `forward()` / `_create_stream_gen()`
|
||||
skeleton plus pipeline/deploy definitions:
|
||||
[references/single-stage-ar.md](references/single-stage-ar.md). For an
|
||||
in-tree reference, look for any single-stage AR model under
|
||||
`vllm_omni/model_executor/models/`, such as MOSS-TTS-Nano.
|
||||
|
||||
**VoxCPM2 is a different pattern** and should not reuse this skeleton — it
|
||||
runs the base LM under vLLM PagedAttention with external side-computation.
|
||||
See `plan/voxcpm2_native_ar_design.md`.
|
||||
|
||||
### Deliverables
|
||||
|
||||
- Model files in `vllm_omni/model_executor/models/<model_name>/`
|
||||
- Registered `pipeline.py` topology
|
||||
- Deploy YAML under `vllm_omni/deploy/`
|
||||
- Working `end2end.py` at `examples/offline_inference/text_to_speech/<model>/end2end.py`
|
||||
- New section in `examples/offline_inference/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/offline_inference/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/` — the hub README is the documented surface and the mkdocs `generate_examples` hook only descends one level into `examples/<category>/`.
|
||||
|
||||
## Phase 3: Online Serving
|
||||
|
||||
**Goal**: Expose the model via `/v1/audio/speech` API endpoint.
|
||||
|
||||
### Steps
|
||||
|
||||
1. **Write one adapter** under `vllm_omni/entrypoints/openai/tts_adapters/`.
|
||||
`serving_speech.py` should not need an edit — detection, stage discovery and
|
||||
dispatch are all derived from what the adapter declares.
|
||||
|
||||
Create `vllm_omni/entrypoints/openai/tts_adapters/your_model.py`:
|
||||
```python
|
||||
@register_tts_adapter
|
||||
class YourModelAdapter(ARTTSAdapter):
|
||||
name = "your_model" # registry key + log label
|
||||
stage_keys = frozenset({"your_stage_key"}) # the deploy yaml's model_stage
|
||||
|
||||
def validate(self, request) -> str | None:
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
return None
|
||||
|
||||
async def build(self, request, sampling_params_list, has_inline_ref_audio):
|
||||
params = {"text": [request.input]}
|
||||
if request.voice is not None:
|
||||
params["voice"] = [request.voice]
|
||||
return PreparedRequest(
|
||||
prompt={"prompt": request.input},
|
||||
tts_params=params,
|
||||
model_type=self.name,
|
||||
)
|
||||
```
|
||||
|
||||
Then add the module to the import block at the bottom of
|
||||
`tts_adapters/__init__.py` so it registers. That import line is the only shared
|
||||
file a new model touches — which is also why the old rebase-conflict hotspot is
|
||||
gone.
|
||||
|
||||
> **Pure-diffusion TTS does not go through adapters yet.** Under
|
||||
> `for_diffusion()`, `create_speech()` routes straight to
|
||||
> `_create_diffusion_speech()` and never calls `validate()`/`build()`.
|
||||
> `DiffusionTTSAdapter` is scaffolding with no production subclass, so logic
|
||||
> placed in one would silently never run. Diffusion-engine models follow the
|
||||
> existing diffusion path; wiring it through adapters is open work (#4855).
|
||||
|
||||
**If a stage key alone cannot identify the model**, declare
|
||||
`model_archs = frozenset({"YourModelForConditionalGeneration"})`; add
|
||||
`arch_identifies_entry_stage = True` when the model owns no stage key at all
|
||||
(Ming dense). For a rule that is not set membership, override `matches()` (see
|
||||
`covo_audio.py`). For a genuine overlap with another adapter, give one an
|
||||
explicit `detect_priority` — `test_tts_detection.py` fails on an unordered
|
||||
overlap. Models that only serve speech in some topologies override
|
||||
`stage_serves_speech()` (see `audex.py`).
|
||||
|
||||
**Reuse shared helpers via `self.ctx.server`** rather than reimplementing them:
|
||||
`_resolve_ref_audio`, `_apply_uploaded_speaker`, `_validate_ref_audio_format`,
|
||||
`_max_instructions_length`. Read a comparable adapter first — `fish_speech.py`
|
||||
(voice cloning), `higgs_audio_v3.py` (parameter-heavy), `moss_tts.py` (family
|
||||
sharing a base class).
|
||||
|
||||
> **Do not add `self._tts_model_type == ...` branches to `serving_speech.py`.**
|
||||
> Older models predate the adapter framework and still have them; they are being
|
||||
> migrated out (RFC #4327, #4855). `tools/pre_commit/check_tts_adapter.py` is a
|
||||
> ratchet on the remaining count and fails the commit if it grows. Behaviour that
|
||||
> no adapter hook can express is a missing hook — propose it on the RFC.
|
||||
|
||||
> **Unused variable rule**: only extract fields in `build()` that are actually
|
||||
> forwarded to the model. Unused extractions fail `ruff F841`. For voice-cloning
|
||||
> fields (`ref_audio` -> `prompt_audio_path`, `ref_text` -> `prompt_text`), add
|
||||
> them to the params and verify they reach the model call.
|
||||
|
||||
2. **Handle model-specific parameters**:
|
||||
- Voice cloning: `ref_audio` encoding and prompt injection
|
||||
- `max_new_tokens` override in sampling params
|
||||
- Model-specific default values
|
||||
3. **Create client scripts**: `speech_client.py`, `run_server.sh`
|
||||
4. **Test all response formats**: wav, mp3, flac, pcm
|
||||
5. **Add Gradio demo**: Interactive web UI with streaming support
|
||||
|
||||
### Voice Cloning Pattern
|
||||
|
||||
```python
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
|
||||
"""Build prompt with reference audio for voice cloning, called from the adapter."""
|
||||
audio_bytes = Path(ref_audio_path).read_bytes()
|
||||
codes = codec.encode(audio_bytes) # Encode on CPU using model's codec (e.g., DAC)
|
||||
token_ids = [code + codec.vocab_offset for code in codes.flatten().tolist()]
|
||||
return [
|
||||
{"role": "system", "content": f"<|voice|>{''.join(chr(t) for t in token_ids)}"},
|
||||
{"role": "user", "content": text},
|
||||
]
|
||||
```
|
||||
|
||||
### Test Case Writing (CI Levels)
|
||||
|
||||
**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md).
|
||||
|
||||
Classify the model's **CI priority** first (high / medium / low). High-priority TTS models are typically those on the integration hot path or listed in tracking issues such as [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); medium and low tiers cover the long tail. When unsure, ask the reviewer which tier applies.
|
||||
|
||||
| Priority | Required test levels | Files & markers |
|
||||
|----------|---------------------|-----------------|
|
||||
| **High** | **L1** unit/logic · **L2** online smoke · **L3** online + offline integration · **L4** feature + performance | See table below |
|
||||
| **Medium** | **L3** online + offline · **L4** feature only | Skip dedicated L1/L2 unless fixing a logic bug |
|
||||
| **Low** | **L4** feature only | One or two `*_expansion.py` parametrized cases |
|
||||
|
||||
**Per-level deliverables (TTS / `pytest.mark.tts`):**
|
||||
|
||||
| Level | Location | Marker | CI pipeline | Notes |
|
||||
|-------|----------|--------|-------------|-------|
|
||||
| **L1** | `tests/model_executor/…`, `tests/entrypoints/openai_api/…`, stage-processor tests | `core_model` + `cpu` | `test-ready.yml` | Prompt assembly, async_chunk helpers, adapter validation — no GPU |
|
||||
| **L2** | `tests/e2e/online_serving/test_{slug}.py` | **`core_model` + `advanced_model`** (both on baseline smoke) + `tts` + `@hardware_test(...)` | `test-ready.yml` (`ready` label) | Default deploy smoke: single `/v1/audio/speech` or offline `OmniRunner` path |
|
||||
| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `tts`) | `test-merge.yml` **or** merged into nightly TTS function job | Streaming, voice clone, batch/queue, async_chunk |
|
||||
| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py`, optional offline expansion | `full_model` + `tts` | `test-nightly.yml` (`:full_moon: TTS · Function Test with L4`) | Feature matrix; perf → `tests/dfx/perf/tests/test_tts.json` |
|
||||
|
||||
**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, single non-streaming `/v1/audio/speech` or equivalent offline path) should carry **both** `@pytest.mark.core_model` **and** `@pytest.mark.advanced_model` on the **same** function so it runs in L2 (`test-ready.yml`, `--run-level core_model`, basic validation) and L3 (`test-merge.yml`, `--run-level advanced_model`, deeper validation) without duplicating the test. In-tree examples: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`.
|
||||
|
||||
Heavier scenarios in the same file use **`advanced_model` only** (streaming, extra languages, concurrency, async_chunk, batch). Example: `test_voice_clone_en_streaming_001` → `advanced_model` only. When migrating L3 to nightly, move those heavier cases into `test_{slug}_expansion.py` with `full_model` and drop the dedicated merge job (see `test_ming_tts_expansion.py`, `test_glm_tts_expansion.py`).
|
||||
|
||||
```python
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_voice_clone_en_non_streaming_001(omni_server, online_client) -> None:
|
||||
online_client.send_audio_speech_request({...})
|
||||
```
|
||||
|
||||
**L4 consolidation:** Prefer parametrized `OmniServerParams` rows (`default`, `async_chunk`, feature flags) in one expansion module rather than many merge-only files ([#1832](https://github.com/vllm-project/vllm-omni/issues/1832)).
|
||||
|
||||
**L4 performance (high-priority models):** Add latency / throughput / stress rows in `tests/dfx/perf/tests/test_tts.json`, or a dedicated `tests/dfx/perf/tests/test_{slug}.json` when the model must not join the shared nightly server matrix before integration lands (see VoxCPM2 / Coqui XTTS pattern). Register the model in `benchmarks/tts/model_configs.yaml` for local `bench_tts.py`. Wire a **separate** `test-nightly.yml` Perf Test step when the JSON is not merged into `test_tts.json` yet.
|
||||
|
||||
**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:**
|
||||
|
||||
- Put `MODEL`, deploy path, vendored `REF_AUDIO_URL`, `get_prompt()`, and inline `request_config` dicts **in each** `test_{slug}.py`, `test_{slug}_expansion.py`, offline `test_{slug}.py`, and L1 `test_{slug}_*.py` as needed.
|
||||
- **Do not** add `tests/helpers/{slug}.py` (or `tests/helpers/{model_name}.py`) to deduplicate constants or request builders across those files. A little duplication is intentional; follow in-tree references such as `tests/e2e/online_serving/test_glm_tts.py` and `tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py`.
|
||||
- `tests/helpers/` is for **repo-wide** harness code only (`mark.py`, `media.py`, `runtime.py`, `stage_config.py`, `assertions.py`, `fixtures/`). Import those; do not extend the tree with per-model modules.
|
||||
|
||||
**Runtime send helpers (`tests/helpers/runtime.py`) — online and offline e2e:**
|
||||
|
||||
| Path | Fixture | Call |
|
||||
|------|---------|------|
|
||||
| Online `/v1/*` | `online_client` | `online_client.send_*_request(request_config)` |
|
||||
| Offline inference | `offline_client` | `offline_client.send_*_request(request_config)` |
|
||||
|
||||
1. **Grep `runtime.py` first** — reuse `send_omni_request`, `send_diffusion_request`, `send_audio_speech_request` (online + offline Qwen-style TTS), `send_single_stage_tts_request` (Coqui XTTS / MOSS-TTS-Nano offline), etc.
|
||||
2. **No matching helper** → add `send_<feature>_request` (or `send_<route>_http_request` for negative/dfx) in **`runtime.py`** with general `assert_*` bundled inside, **then** call it from the test.
|
||||
3. **Test file** holds `request_config` dicts only — not `omni.generate`, not `_collect_audio()`, not raw HTTP/SDK.
|
||||
4. Wire `tests/helpers/runtime.py` into Buildkite `source_file_dependencies` when you add helpers.
|
||||
|
||||
See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers** for full tables and exceptions.
|
||||
|
||||
### Deliverables
|
||||
|
||||
- One adapter file under `tts_adapters/` plus its line in the package import block
|
||||
- Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
|
||||
- Gradio demo with streaming and voice cloning UI in the same dir
|
||||
- E2E tests per **Test Case Writing (CI Levels)** above (priority tier determines L1–L4 scope)
|
||||
- Buildkite wired per level: `test-ready.yml` (L1/L2), `test-merge.yml` or nightly function job (L3), `test-nightly.yml` (L4) — see [vllm-omni-test skill](../vllm-omni-test/SKILL.md)
|
||||
- New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
|
||||
|
||||
### E2E test pitfalls to avoid
|
||||
|
||||
- **One `OmniServerParams` set per file.** `omni_server` is module-scoped; a second
|
||||
id in the same file forces mid-module teardown/restart and exposes startup
|
||||
races (`APIConnectionError` on the first request post-restart). Split variants
|
||||
into separate files instead.
|
||||
- **No external URL fetches from the server.** CI and some dev hosts can't
|
||||
reach `raw.githubusercontent.com` over TLS. Inline ref audio as
|
||||
`data:audio/wav;base64,...`; the serving layer accepts both URL and data URL.
|
||||
- **Use the harness readiness gate.** The fixture waits for HTTP 200 on
|
||||
`/health`; don't add `time.sleep` in tests. If warmup is incomplete, make
|
||||
`/health` return non-200 until you're actually ready.
|
||||
- **Mark tests per the CI Levels table** — baseline smoke: `core_model` + `advanced_model`; heavier cases: `advanced_model` only; L4 expansion: `full_model`
|
||||
- **No per-model helper modules** — do not create `tests/helpers/{slug}.py`; keep constants and `request_config` payloads in the test file
|
||||
- **Online and offline e2e go through `runtime.py`** — `online_client.send_audio_speech_request` (online); `offline_client.send_audio_speech_request` (Qwen-style offline) or `send_single_stage_tts_request` (single-stage offline). Add a new `send_*_request` in `runtime.py` when none fits; do not embed `omni.generate` or HTTP in tests
|
||||
|
||||
## Phase 4: Async Chunk (Streaming)
|
||||
|
||||
**Goal**: Enable inter-stage streaming so audio chunks are produced while AR generation continues.
|
||||
|
||||
### Steps
|
||||
|
||||
1. **Update the pipeline topology** so the producing stage declares its async
|
||||
handoff processor with `async_chunk_process_next_stage_input_func`.
|
||||
2. **Update the deploy YAML** to enable async chunk and configure the connector:
|
||||
```yaml
|
||||
async_chunk: true
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
codec_streaming: true
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
```
|
||||
3. **Implement chunk handling in Stage 1**:
|
||||
- Accept partial input (chunk of codec codes)
|
||||
- Handle left context for smooth audio boundaries
|
||||
- Return partial audio in `OmniOutput`
|
||||
4. **Test streaming**:
|
||||
- Verify audio quality matches non-streaming output
|
||||
- Check for artifacts at chunk boundaries
|
||||
- Measure TTFA (time to first audio)
|
||||
5. **Update online serving** to support `stream=true` with PCM output
|
||||
|
||||
### Streaming Architecture
|
||||
|
||||
```
|
||||
Stage 0 (AR) Stage 1 (Decoder)
|
||||
| |
|
||||
|-- chunk 0 (25 frames) ------> decode -> audio chunk 0 -> client
|
||||
|-- chunk 1 (25 frames) ------> decode -> audio chunk 1 -> client
|
||||
|-- chunk 2 (25 frames) ------> decode -> audio chunk 2 -> client
|
||||
...
|
||||
```
|
||||
|
||||
### Key Considerations
|
||||
|
||||
- **Left context overlap**: Prevents audible artifacts at chunk boundaries
|
||||
- **Hop length matters**: `context_audio_samples = context_frames * hop_length`
|
||||
- **First chunk latency**: Can use larger initial chunk for better quality, then smaller chunks
|
||||
|
||||
### Deliverables
|
||||
|
||||
- Updated pipeline async handoff processor and deploy config with
|
||||
`async_chunk: true`
|
||||
- Smooth streaming audio without boundary artifacts
|
||||
- TTFA metrics
|
||||
|
||||
## Phase 5: CUDA Graph Acceleration
|
||||
|
||||
**Goal**: Capture the AR loop as a CUDA graph for significant speedup.
|
||||
|
||||
### Steps
|
||||
|
||||
1. **Identify the hot loop**: The AR decoding loop that runs N steps per token
|
||||
2. **Create static buffers**:
|
||||
- KV caches with fixed max sequence length
|
||||
- Pre-built causal masks and position tensors per step
|
||||
- Static input/output tensors
|
||||
3. **Implement graph capture**:
|
||||
- Warm up with real data
|
||||
- Capture the forward pass
|
||||
- Replay with updated inputs
|
||||
4. **Handle constraints**:
|
||||
- Use `torch.argmax` instead of `torch.multinomial` (graph-safe)
|
||||
- Fixed batch size (fall back to eager for other sizes)
|
||||
- No dynamic control flow inside the graph
|
||||
|
||||
See [references/cuda-graph-example.md](references/cuda-graph-example.md) for
|
||||
a worked skeleton (Qwen3-TTS code predictor, 16-step AR loop), performance
|
||||
expectations (3–5× on the graphed component for fixed batch_size=1), and the
|
||||
graph-safety constraints you must honor inside the captured region.
|
||||
|
||||
### Deliverables
|
||||
|
||||
- CUDA graph implementation for the AR hot loop
|
||||
- Benchmark script comparing eager vs graph performance
|
||||
- Documentation of constraints and fallback behavior
|
||||
|
||||
## Phase 6: Pre-commit and DCO
|
||||
|
||||
**Goal**: Every commit passes `pre-commit` lint and carries a DCO
|
||||
`Signed-off-by` line that matches the author email.
|
||||
|
||||
- Install hooks once: `pre-commit install`.
|
||||
- Run `pre-commit run --files <changed-files>` before every push; accept any
|
||||
auto-fixes, stage, re-commit.
|
||||
- Sign every commit with `git commit -s`. DCO checks that author email and
|
||||
`Signed-off-by` email match — `git config user.email` must match your
|
||||
GitHub account email.
|
||||
|
||||
Common pre-commit failures, recovery commands for missing sign-off, and the
|
||||
full `pre-commit run` invocation for a TTS model:
|
||||
[references/precommit-dco.md](references/precommit-dco.md).
|
||||
|
||||
## Integration Checklist
|
||||
|
||||
Use this checklist when integrating a new TTS model:
|
||||
|
||||
### Cross-Cutting Invariants (verify at end of every phase)
|
||||
- [ ] I1: `forward()` docstring states cumulative vs delta; consolidation path audited end-to-end
|
||||
- [ ] I2: Tests / examples / benchmarks never use `dict.get(a) or dict.get(b)` on tensor values; list form handled
|
||||
- [ ] I3: No `.item()` / `.cpu()` / Python branch on tensor values inside per-step loops
|
||||
- [ ] I4: Offline RTF, browser streaming playback, and concurrent-request smoke test all pass
|
||||
- [ ] I5: Any cross-step cache keyed by `_omni_req_id`; entries freed when the request finishes
|
||||
|
||||
### Phase 1: HF Reference
|
||||
- [ ] Reference model runs and produces correct audio
|
||||
- [ ] Architecture documented (stages, codebooks, tokens, sample rate)
|
||||
- [ ] Reference audio samples saved for comparison
|
||||
|
||||
### Phase 2: Stage Separation
|
||||
- [ ] Model registered in `registry.py`
|
||||
- [ ] Config classes created with `model_type` registration
|
||||
- [ ] Stage 0 (AR) implemented and generates correct tokens
|
||||
- [ ] Stage 1 (Decoder) produces correct audio from tokens — dtype float32 for codec decoder
|
||||
- [ ] AR stage `max_num_seqs` ≥ 4 in the production deploy config unless the model has a tested lower limit
|
||||
- [ ] Optional dependency fallbacks handled at `load_weights()` time (torchaudio/soundfile/etc.)
|
||||
- [ ] Streaming: codec codes accumulated across AR steps (not reset per step)
|
||||
- [ ] Streaming: delta audio emitted per chunk, not full re-decoded waveform
|
||||
- [ ] Streaming: all `forward()` return paths emit `model_outputs`
|
||||
- [ ] Streaming: per-request state keyed by request ID (not shared across requests)
|
||||
- [ ] Streaming: codec tensors moved to codec decoder device before decode
|
||||
- [ ] Pipeline topology defined in `pipeline.py` and registered in `OMNI_PIPELINES`
|
||||
- [ ] Deploy YAML created under `vllm_omni/deploy/`
|
||||
- [ ] `end2end.py` produces audio matching reference quality
|
||||
- [ ] README.md written
|
||||
|
||||
### Phase 3: Online Serving
|
||||
- [ ] Adapter written under `tts_adapters/` and registered in the import block
|
||||
- [ ] Only extract params in `build()` that are forwarded to the model call (ruff F841)
|
||||
- [ ] Prompt builder handles text input correctly
|
||||
- [ ] Voice cloning works (if supported)
|
||||
- [ ] All response formats work (wav, mp3, flac, pcm)
|
||||
- [ ] Client scripts and server launcher created
|
||||
- [ ] E2E tests added per model priority tier (see **Test Case Writing (CI Levels)**)
|
||||
- [ ] Buildkite entries match level: `test-ready.yml` / `test-merge.yml` or nightly TTS job / `test-nightly.yml`
|
||||
- [ ] Gradio demo working
|
||||
- [ ] Documentation added (offline + online docs, nav, supported models)
|
||||
|
||||
### Phase 4: Async Chunk
|
||||
- [ ] Pipeline declares the async handoff processor
|
||||
- [ ] Deploy config sets `async_chunk: true` and connector chunk parameters
|
||||
- [ ] Stage 1 handles partial chunks correctly
|
||||
- [ ] No audio artifacts at chunk boundaries
|
||||
- [ ] Streaming via API (`stream=true`) works
|
||||
- [ ] TTFA measured and acceptable
|
||||
|
||||
### Phase 5: CUDA Graph
|
||||
- [ ] Hot loop identified and profiled
|
||||
- [ ] Static buffers allocated
|
||||
- [ ] Graph captured and replays correctly
|
||||
- [ ] Benchmark shows meaningful speedup
|
||||
- [ ] Fallback to eager works for unsupported configs
|
||||
|
||||
### Phase 6: Pre-commit and DCO
|
||||
- [ ] `pre-commit run --files <changed>` passes before every push
|
||||
- [ ] Every commit has `Signed-off-by` matching the author email (`git commit -s`)
|
||||
- [ ] `git config user.email` matches the email registered on your GitHub account
|
||||
- [ ] Hook list, failure table, and DCO recovery: [references/precommit-dco.md](references/precommit-dco.md)
|
||||
|
||||
## References
|
||||
|
||||
In-skill references (details split out of the main body):
|
||||
|
||||
- [references/single-stage-ar.md](references/single-stage-ar.md) — full `forward()` / generator skeleton for the MOSS-TTS-Nano-style pattern
|
||||
- [references/optional-deps.md](references/optional-deps.md) — torchaudio / torchcodec fallback pattern
|
||||
- [references/cuda-graph-example.md](references/cuda-graph-example.md) — Qwen3-TTS code-predictor CUDA graph skeleton
|
||||
- [references/precommit-dco.md](references/precommit-dco.md) — full pre-commit invocation, failure table, DCO recovery
|
||||
|
||||
Project docs and adjacent skills:
|
||||
|
||||
- [TTS audio skill](../vllm-omni-audio-tts/SKILL.md) — supported models and usage
|
||||
- [Fish Speech integration](../vllm-omni-audio-tts/references/fish-speech.md) — complete example of Phases 1–3
|
||||
- [Qwen3-TTS reference](../vllm-omni-audio-tts/references/qwen-tts.md) — complete example of all 5 phases
|
||||
- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
|
||||
- [Adding a TTS model (developer guide)](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/model/adding_tts_model.md)
|
||||
- `plan/voxcpm2_native_ar_design.md` — VoxCPM2's vLLM-native AR + side-computation pattern (distinct from the generator-based single-stage described above)
|
||||
@@ -1,42 +0,0 @@
|
||||
# CUDA Graph Example: Qwen3-TTS Code Predictor
|
||||
|
||||
Reference sketch for capturing the 16-step code-predictor AR loop as a single
|
||||
CUDA graph. Adapt the shapes, number of steps, and KV-head layout to your
|
||||
model.
|
||||
|
||||
```python
|
||||
import torch
|
||||
|
||||
class CodePredictorGraph:
|
||||
"""Captures the 16-step code predictor AR loop as a single CUDA graph."""
|
||||
|
||||
def setup_graph(self, device: torch.device, kv_heads: int = 4, head_dim: int = 64):
|
||||
self.num_steps = 16
|
||||
self.kv_cache = torch.zeros(1, kv_heads, self.num_steps, head_dim, device=device)
|
||||
self.positions = torch.arange(self.num_steps, device=device)
|
||||
self.causal_mask = torch.tril(torch.ones(self.num_steps, self.num_steps, device=device))
|
||||
self.input_buf = torch.zeros(1, 1, kv_heads * head_dim, device=device)
|
||||
self.output_buf = torch.zeros(1, self.num_steps, device=device, dtype=torch.long)
|
||||
# Warm up, then: self.graph = torch.cuda.CUDAGraph(); self.graph.capture(...)
|
||||
|
||||
def run_graph(self, initial_input: torch.Tensor) -> torch.Tensor:
|
||||
self.input_buf.copy_(initial_input)
|
||||
self.graph.replay()
|
||||
return self.output_buf.clone()
|
||||
```
|
||||
|
||||
## Performance expectations (Qwen3-TTS code predictor)
|
||||
|
||||
- **3–5× speedup** on the graphed component.
|
||||
- Effective only for fixed batch sizes (typically `batch_size=1`).
|
||||
- Fall back to eager for any shape/config that wasn't captured — do not try
|
||||
to recapture per request.
|
||||
|
||||
## Graph-safety constraints
|
||||
|
||||
- `torch.argmax` instead of `torch.multinomial`.
|
||||
- Fixed batch size.
|
||||
- No Python control flow that branches on tensor values inside the captured
|
||||
region (use `torch.where` / masks).
|
||||
- No `.item()`, `.cpu()`, `.tolist()` — each would break the capture or
|
||||
cause a GPU→CPU sync during replay.
|
||||
@@ -1,47 +0,0 @@
|
||||
# Optional Dependency Handling
|
||||
|
||||
Models that rely on `torchaudio`, `torchcodec`, `soundfile`, or other optional
|
||||
packages must handle the missing-package case at import time, not at call
|
||||
time. Failing to do this causes cryptic errors only on environments without
|
||||
the optional package — after the model is already deployed.
|
||||
|
||||
## Pattern (used in MOSS-TTS-Nano)
|
||||
|
||||
```python
|
||||
def _patch_torchaudio_load() -> None:
|
||||
"""Fallback torchaudio.load/save to soundfile if torchcodec is unavailable."""
|
||||
try:
|
||||
import torchcodec # noqa: F401
|
||||
return # torchcodec present, torchaudio works as-is
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
import soundfile as sf
|
||||
import torchaudio
|
||||
|
||||
def _sf_load(path, **kwargs):
|
||||
data, sr = sf.read(str(path), dtype="float32", always_2d=True)
|
||||
return torch.from_numpy(data).T, sr
|
||||
|
||||
torchaudio.load = _sf_load
|
||||
# patch .save similarly if needed
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
||||
- Mirror the full signature of the replaced function. `torchaudio.load`
|
||||
accepts `frame_offset`, `num_frames`, `normalize`, `channels_first`,
|
||||
`format` — missing any of them causes `TypeError` from calling code.
|
||||
- Catch `except Exception`, not just `ImportError`. `import torchaudio`
|
||||
itself can fail with non-`ImportError` errors on broken installs. Log the
|
||||
exception type and message (`logger.warning("torchaudio probe failed: %s: %s",
|
||||
type(exc).__name__, exc)`) before falling back, so unrelated errors are not
|
||||
silently swallowed.
|
||||
- Call the patch function at the top of `load_weights()` before loading any
|
||||
audio assets. Do not call it at module import time.
|
||||
|
||||
## Reference implementation
|
||||
|
||||
Any in-tree model that patches `torchaudio.load` in its `load_weights()` —
|
||||
e.g. MOSS-TTS-Nano's `modeling_moss_tts_nano.py` once that integration
|
||||
lands.
|
||||
@@ -1,66 +0,0 @@
|
||||
# Pre-commit and DCO
|
||||
|
||||
Every commit must pass `pre-commit` lint and carry a `Signed-off-by` line
|
||||
that matches the commit author email.
|
||||
|
||||
## Pre-commit
|
||||
|
||||
Install hooks once:
|
||||
|
||||
```bash
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
Run before every push on the files you changed:
|
||||
|
||||
```bash
|
||||
pre-commit run --files \
|
||||
vllm_omni/model_executor/models/<model_name>/*.py \
|
||||
vllm_omni/entrypoints/openai/serving_speech.py \
|
||||
vllm_omni/model_executor/models/registry.py \
|
||||
tests/e2e/offline_inference/test_<model_name>.py \
|
||||
tests/e2e/online_serving/test_<model_name>.py
|
||||
```
|
||||
|
||||
When pre-commit **modifies files** (ruff format auto-fix), it exits non-zero
|
||||
but the changes are correct — stage the modified files and re-commit.
|
||||
|
||||
| Failure | Root cause | Fix |
|
||||
|---------|-----------|-----|
|
||||
| `ruff F841` | Variable extracted but never forwarded to model call | Remove the extraction or wire it through |
|
||||
| `ruff E402` | Import added below function definitions | Move to top-level import block |
|
||||
| `ruff format` | Line length, spacing, quote style | Accept auto-fix, stage, re-commit |
|
||||
| `check-spdx-header` | Missing header, or copyright still says `vLLM project` | Two-line Omni header (`vLLM-Omni project`); restage the rewrite |
|
||||
| `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; use `vllm.transformers_utils.repo_utils`. Do **not** add the file to `allowed_files` without review |
|
||||
| `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform` / `OmniPlatform`; do not grow `ALLOWED_FILES` without review |
|
||||
| `check-tts-adapter-migration` | New `self._tts_model_type` branch in `serving_speech.py` | Put per-model logic in `tts_adapters/`. Lower `MAX_MODEL_TYPE_BRANCHES` when removing branches; do not raise it without review |
|
||||
| `check-mark` | New `tests/**/test_*.py` missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
|
||||
| `mypy-3.10` | Type error on changed `vllm_omni/` files | Fix the types; model trees are excluded. Extra versions: `pre-commit run --hook-stage manual mypy-3.12` |
|
||||
| `markdownlint-cli2` | Docs/README markdown lint | Accept auto-fix under `docs/` / `recipes/` / root README |
|
||||
| `check-buildkite` | Invalid `.buildkite/*.yml` | Fix the YAML; do not grow `SKIP_FILES` without review |
|
||||
| `shellcheck` | No `shellcheck` on PATH, or script warning | `apt-get`/`dnf`/`brew install shellcheck`, or `shellcheck.exe` on Windows PATH |
|
||||
|
||||
Canonical policy, CI `SKIP` list, and allowlist locations:
|
||||
[docs/contributing/README.md](../../../docs/contributing/README.md#linting).
|
||||
|
||||
## DCO sign-off
|
||||
|
||||
Every commit must carry `Signed-off-by: Your Name <your@email.com>`. Use
|
||||
`-s`:
|
||||
|
||||
```bash
|
||||
git commit -s -m "feat(<model>): add <Model> TTS support"
|
||||
```
|
||||
|
||||
Or set it permanently: `git config format.signOff true`.
|
||||
|
||||
The DCO check verifies that the commit author email matches the
|
||||
`Signed-off-by` line. Confirm `git config user.email` matches your GitHub
|
||||
account email before committing.
|
||||
|
||||
Fix a missing or mismatched sign-off on the latest commit:
|
||||
|
||||
```bash
|
||||
git commit --amend -s --no-edit
|
||||
git push origin <branch> --force-with-lease
|
||||
```
|
||||
@@ -1,144 +0,0 @@
|
||||
# Single-Stage AR Pattern
|
||||
|
||||
When the upstream model cannot be cleanly split into an AR stage and a separate
|
||||
decoder (e.g. MOSS-TTS-Nano, or any model that bundles AR + codec via an
|
||||
`inference_stream()` generator), run the whole pipeline inside a single AR
|
||||
worker that yields audio chunks per request.
|
||||
|
||||
This is distinct from VoxCPM2's pattern, which also runs in a single stage but
|
||||
uses vLLM's native PagedAttention on the base language model with diffusion /
|
||||
VAE side-computation outside vLLM — see
|
||||
`plan/voxcpm2_native_ar_design.md` for that variant.
|
||||
|
||||
## Implementation
|
||||
|
||||
1. **Single model file** — load both AR LM and codec inside
|
||||
`modeling_<model>.py`.
|
||||
2. **Load weights in `load_weights()`**, not `__init__()` — vLLM initializes
|
||||
distributed state before any CUDA allocations.
|
||||
3. **Stream via a per-request generator** stored in `self._stream_gens`:
|
||||
|
||||
```python
|
||||
class YourModelForCausalLM(nn.Module):
|
||||
def __init__(self, *, vllm_config, prefix=""):
|
||||
super().__init__()
|
||||
self._lm = None # populated in load_weights()
|
||||
self._stream_gens: dict = {} # request_key → generator
|
||||
|
||||
def load_weights(self, weights):
|
||||
# Load self._lm here, after vLLM distributed init
|
||||
...
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids,
|
||||
positions,
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=None,
|
||||
runtime_additional_information: list[dict] | None = None, # one dict per request
|
||||
**kwargs,
|
||||
) -> OmniOutput:
|
||||
infos = runtime_additional_information or [{}]
|
||||
# Skip dummy/profiling calls
|
||||
if not runtime_additional_information or all(i.get("_is_dummy") for i in infos):
|
||||
self._ar_emit_stop_token = True
|
||||
return OmniOutput(...) # return empty outputs
|
||||
|
||||
outputs, last_flags = [], []
|
||||
for info in infos:
|
||||
request_key = str(info.get("_omni_req_id", "0")) # per-request ID from vLLM
|
||||
if request_key not in self._stream_gens:
|
||||
self._stream_gens[request_key] = self._create_stream_gen(info)
|
||||
try:
|
||||
chunk, is_last = next(self._stream_gens[request_key])
|
||||
except StopIteration:
|
||||
chunk, is_last = torch.zeros(0), True
|
||||
if is_last:
|
||||
del self._stream_gens[request_key]
|
||||
outputs.append(chunk)
|
||||
last_flags.append(is_last)
|
||||
|
||||
self._ar_emit_stop_token = all(last_flags)
|
||||
return OmniOutput(multimodal_outputs={"model_outputs": outputs, ...})
|
||||
|
||||
def _create_stream_gen(self, info: dict):
|
||||
"""Yield (waveform_tensor, is_last) tuples from inference_stream()."""
|
||||
for event in self._lm.inference_stream(...):
|
||||
if event["type"] == "audio":
|
||||
yield event["waveform"], False
|
||||
elif event["type"] == "result":
|
||||
# Fallback: some models emit a single "result" event instead of
|
||||
# incremental "audio" events — handle both paths
|
||||
yield event.get("waveform", torch.zeros(0)), True
|
||||
return
|
||||
yield torch.zeros(0), True
|
||||
|
||||
def compute_logits(self, hidden_states, sampling_metadata):
|
||||
# Emit EOS only after the last chunk so the AR scheduler ends the request
|
||||
...
|
||||
```
|
||||
|
||||
## Key points
|
||||
|
||||
- `runtime_additional_information` is the correct parameter name (not
|
||||
`**kwargs`) — it carries one dict per request in the batch.
|
||||
- The request ID is `info.get("_omni_req_id")` — set by vLLM, not by user code.
|
||||
- Handle both `"audio"` (incremental) and `"result"` (final combined) event
|
||||
types from upstream models.
|
||||
|
||||
## Pipeline and deploy config
|
||||
|
||||
Define the fixed single-stage topology in `pipeline.py`:
|
||||
|
||||
```python
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
YOUR_TTS_PIPELINE = PipelineConfig(
|
||||
model_type="your_model_name",
|
||||
default_deploy_config_name="your_model_name.yaml",
|
||||
model_arch="YourModelForCausalLM",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="your_model_name",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="audio",
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
),
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
Register it in `vllm_omni/config/pipeline_registry.py`. Keep placement and
|
||||
runtime sizing in `vllm_omni/deploy/your_model_name.yaml`:
|
||||
|
||||
```yaml
|
||||
async_chunk: false
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
max_num_seqs: 4
|
||||
gpu_memory_utilization: 0.8
|
||||
```
|
||||
|
||||
Do not move `execution_type`, output ownership, tokenizer ownership, or model
|
||||
architecture into the deploy YAML; those are pipeline topology.
|
||||
|
||||
## Lint discipline
|
||||
|
||||
Only extract variables from `additional_information` that you actually
|
||||
forward to the model call — unused extractions trip `ruff F841` in
|
||||
pre-commit.
|
||||
|
||||
## Reference implementation
|
||||
|
||||
Use `vllm_omni/model_executor/models/moss_tts_nano/pipeline.py` and
|
||||
`vllm_omni/deploy/moss_tts_nano.yaml` as the in-tree reference.
|
||||
@@ -1,510 +0,0 @@
|
||||
---
|
||||
name: diffusion-perf-opt
|
||||
description: Diagnose and optimize vLLM Omni diffusion workloads, especially Wan/Qwen/Flux-style image and video generation. Use when Codex is asked to analyze profiling traces, choose parallel strategies, inspect torch profiler trace.json or trace.json.gz timelines, estimate optimization ROI, investigate GPU idle/free bubbles, compare USP/CFG/HSDP/VAE parallelism, or design operator/host/quantization optimizations for vLLM Omni.
|
||||
---
|
||||
|
||||
# vLLM Omni Performance Optimization
|
||||
|
||||
Use this skill to run a disciplined optimization loop for vLLM Omni diffusion workloads. Keep two ideas separate: real performance baselines are collected with low overhead, while torch profiler traces are diagnostic artifacts and may distort latency.
|
||||
|
||||
## First Questions
|
||||
|
||||
Before proposing changes, ask for the optimization scene if it is not already known:
|
||||
|
||||
- GPU model, card count, topology, and whether NVLink is present.
|
||||
- Model and pipeline, for example Wan2.2 I2V A14B.
|
||||
- User workload: resolution, frames, steps, batch/concurrency, CFG scales, prompt/image inputs.
|
||||
- Three runnable commands for each model/workload, or scripts that generate them:
|
||||
- **Server startup command**: exact `vllm serve` command, environment variables, model path, port, parallelism flags, profiler flags, and precision/compile settings.
|
||||
- **User request command**: exact single-request client command (for example curl or Python client) with fixed prompt/media/seed/size/steps. Use this to validate correctness and collect per-request stage timings.
|
||||
- **User benchmark command**: exact repeatable benchmark command or script with warmup count, measured iteration count, concurrency/batch policy, output directory, and summary format.
|
||||
- Current enabled strategies: USP/SP, CFG parallel, HSDP/FSDP, VAE patch parallel, torch.compile, profiler options.
|
||||
- Optimization target: latency, throughput, memory, cost, or quality-preserving speed.
|
||||
- Precision/quality tolerance: bf16/fp8/quantization/sparsity/approximate attention allowed or not.
|
||||
- Quality validation method: exact-output comparison when feasible, or metrics
|
||||
such as SSIM, PSNR, LPIPS, cosine similarity, MAE, MSE, temporal flicker
|
||||
checks, baseline self-run variance, and reviewable output artifacts.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. **Freeze the measurement protocol and commands.**
|
||||
- Before analyzing parallel strategies or traces, establish the three commands for the target model/workload: server startup, single user request, and user benchmark.
|
||||
- Prefer checked-in scripts or one-off shell scripts over free-form commands in chat. The scripts should make workload variables explicit, including model path, port/base URL, prompt/media inputs, resolution, frames, steps, seed, warmup, measured iterations, concurrency, and output path.
|
||||
- Measurement reports must preserve the concrete process, not just the final numbers. For each tested configuration, write down the server startup command, the user request command, the user benchmark command or polling loop, the final server response metrics, and the client-side timing/HTTP result.
|
||||
- Present all duration and latency values in **milliseconds (ms)** in tables and summaries. If an API returns seconds, convert to ms before reporting. Keep raw units only inside quoted raw logs or source snippets.
|
||||
- Keep profiler modes separate in the command set:
|
||||
- Baseline/benchmark commands should avoid torch profiler and stack collection.
|
||||
- Diagnostic commands may enable torch profiler after a baseline identifies a bottleneck.
|
||||
- If any of the three commands is missing, ask for it or create a proposed script before proceeding.
|
||||
- Make clear which command is authoritative for correctness validation and which command is authoritative for performance numbers.
|
||||
|
||||
2. **Collect a real baseline.**
|
||||
- Disable PyTorch profiler and stack collection.
|
||||
- Disable or fix torch.compile state so A/B is fair.
|
||||
- Avoid `--enforce-eager` for production-speed baselines unless eager is the target.
|
||||
- Prefer `--log-stats` and `--enable-diffusion-pipeline-profiler` for low-overhead stage timing. PR 3069 has the relevant metrics/log-stats changes; if local code does not include them, fetch or cherry-pick the minimal metrics changes rather than merging unrelated PR drift.
|
||||
- Run warmup requests and exclude first-request lazy init.
|
||||
|
||||
3. **Model the parallel strategy before testing.**
|
||||
- Estimate compute and communication for self-attention, cross-attention, FFN, CFG, VAE encode/decode, and HSDP.
|
||||
- Select a small candidate matrix rather than exhaustively testing everything.
|
||||
- Typical 2-card candidates: `USP=2`, `CFG=2`, `USP=1/HSDP on-off`, VAE parallel on-off if memory allows.
|
||||
- Typical 4-card candidates: `USP=4`, `CFG=2 x USP=2`, `USP=2 x HSDP`, VAE parallel on-off.
|
||||
- Typical 8-card candidates for official CFG-enabled video diffusion models:
|
||||
- Primary candidate: `CFG=2 x USP=4` with VAE patch parallel across all 8 ranks.
|
||||
- Compare against: `CFG=1 x USP=8` to test whether larger sequence parallel groups beat CFG branch parallelism.
|
||||
- Isolate VAE parallelism: keep the DiT strategy fixed and compare VAE patch parallel on/off or different VAE patch sizes.
|
||||
- If long-video `diffuse` remains dominant and Ulysses all-to-all is suspected, test a hybrid sequence strategy such as `CFG=2 x USP=2 x Ring=2`.
|
||||
- Test HSDP off only if the model fits without it; treat HSDP primarily as a memory strategy until A/B proves otherwise.
|
||||
- Prefer USP/SP for long video token sequences; prefer CFG parallel when CFG doubles transformer forwards and sequence length is modest.
|
||||
- Convert workload shape to latent/patch token counts before choosing candidates. For video models, record frames, latent frames, latent HxW, patch size, approximate token count, and which stages should scale with the token count.
|
||||
|
||||
4. **Search for the best parallel configuration.**
|
||||
- Start with a small matrix that answers one question per comparison:
|
||||
- `CFG parallel vs larger USP`: compare `CFG=2 x USP=world/2` against `CFG=1 x USP=world` for CFG-enabled workloads.
|
||||
- `VAE patch parallel value`: compare the best DiT strategy with VAE patch parallel enabled and disabled.
|
||||
- `HSDP cost`: compare HSDP on/off only when both configurations fit memory.
|
||||
- `Ulysses vs Ring`: test a Ring/Ulysses hybrid only after long-sequence `diffuse` is confirmed dominant.
|
||||
- For each configuration, create or record a stable config id, for example `A_cfg2_usp4_vaepp8_hsdp_tiling`.
|
||||
- For each config id, capture:
|
||||
- exact server command and environment variables,
|
||||
- observed distributed setup from logs, such as SP groups, CFG groups, HSDP shard/replicate sizes, VAE patch size,
|
||||
- exact request command for every scenario,
|
||||
- final server response metrics and client-side elapsed time,
|
||||
- output artifact paths and any failed/empty responses.
|
||||
- Use one warmup request per scenario, then at least three measured repeats for the shortlisted config. For exploratory matrix pruning, one measured request is acceptable only if the margin is large; label it as one-shot.
|
||||
- Compare configurations by stage, not only end-to-end latency. A configuration can improve `diffuse` while hurting `vae.decode`; record both effects.
|
||||
- Select the best config only after checking the target metric, dominant stages, memory headroom, and output correctness.
|
||||
|
||||
5. **Run targeted A/B tests.**
|
||||
- Change one variable per test.
|
||||
- Keep model, input, seed, request parameters, GPU placement, and warmup policy fixed.
|
||||
- Record latency, stage timings, memory, output quality, and logs.
|
||||
- Report comparison tables in ms. Include at least end-to-end client time, server `inference_time`, server stage generation time if available, `vae.encode`, `diffuse`, `vae.decode`, and peak memory.
|
||||
|
||||
6. **Enforce the quality and precision gate for every optimization.**
|
||||
- Do not mark an optimization ready until quality is checked on the same
|
||||
workload: model, prompt/media, seed, shape, steps, scheduler, dtype,
|
||||
backend, parallelism, and output encoding.
|
||||
- For math-preserving changes, require exact/near-exact agreement or show
|
||||
that differences are within baseline self-run variance.
|
||||
- For precision, quantization, approximate attention, sparse/custom kernels,
|
||||
or backend changes, include reviewable artifacts and metrics such as SSIM,
|
||||
PSNR, LPIPS, cosine similarity, MAE, MSE, and temporal flicker checks.
|
||||
- If quality tolerance is not stated, default to quality-preserving behavior.
|
||||
Failed or inconclusive quality validation blocks a ready-to-merge claim.
|
||||
|
||||
7. **Collect diagnostic trace only after narrowing hypotheses.**
|
||||
- Use torch profiler for a small number of requests.
|
||||
- Run two separate diagnostic traces instead of mixing concerns:
|
||||
- **Operator/shape trace**: enable `torch_profiler_record_shapes=True` and keep stack collection disabled. Use this to rank CUDA kernels, NCCL collectives, attention/MLP/norm/RoPE work, and shape-specific hot operators.
|
||||
- **Host-stack trace**: enable `torch_profiler_with_stack=True` and normally keep shape collection disabled. Use this to map CPU/Python host gaps, synchronization points, scheduler paths, and request handling overhead.
|
||||
- Keep both trace commands and reports separate from baseline/benchmark commands. Torch profiler latency is diagnostic only and must not be used as the final latency claim.
|
||||
- Prefer profiling only the narrowed dominant scenario, for example the highest-resolution/video-length case where `diffuse` dominates.
|
||||
- If profiler endpoints are available, run one warmup request first, then call `/start_profile`, run one profiled request, and call `/stop_profile`. This keeps model initialization and warmup out of the diagnostic trace.
|
||||
- Start by analyzing rank 0 only. Expand to more ranks only if rank 0 suggests imbalance, unclear GPU idle/free bubbles, high NCCL wait, server timing mismatch, CFG branch imbalance, or USP group stragglers.
|
||||
- Diagnostic reports must be written to disk and preserve: server command, profiler config, warmup/request/polling commands, trace artifact paths, rank analyzed, analyzer output or summary, and the decision about whether additional ranks are necessary.
|
||||
- Analyze both rank-level balance and device-level free bubbles when additional ranks are opened.
|
||||
|
||||
8. **Analyze host, communication, and operators.**
|
||||
- Find GPU idle/free intervals and map each large gap to the enclosing CPU/Python code.
|
||||
- Separate real GPU idle from profiler overhead such as CUPTI `Command Buffer Full`.
|
||||
- Compare NCCL kernel time to user annotations; annotations can overcount nested intervals.
|
||||
- Rank operator work by total CUDA time and by repeated small-kernel launch count.
|
||||
|
||||
9. **Produce an optimization plan.**
|
||||
- Classify candidates as P0/P1/P2.
|
||||
- For each candidate, state necessity, expected benefit, implementation path, validation plan, and quality risk.
|
||||
- Include a concrete quality/precision validation gate for each candidate.
|
||||
- Do not implement high-risk operator rewrites before proving the operator is a bottleneck for the target shapes.
|
||||
- End the plan with a user-facing candidate selection table. The assistant
|
||||
should not automatically choose a risky optimization just because it is
|
||||
technically possible. Present the options clearly and let the user decide
|
||||
which item is worth implementing based on latency target, engineering
|
||||
budget, memory headroom, and quality tolerance.
|
||||
- Organize the plan by optimization layer, not by a flat list of ideas:
|
||||
- host/runtime optimization,
|
||||
- measurement/benchmark reliability,
|
||||
- parallelism and communication,
|
||||
- VAE encode/decode and media pre/post processing,
|
||||
- operator fusion and layout cleanup,
|
||||
- attention main-path optimization,
|
||||
- algorithmic, precision, or approximation changes.
|
||||
- For each layer, tie every candidate back to evidence from baseline
|
||||
metrics, diagnostic trace, source code, or output quality requirements.
|
||||
Do not list generic optimizations without a trace or workload reason.
|
||||
|
||||
## Priority Rules
|
||||
|
||||
- **P0:** low risk, likely useful, or required for trustworthy measurement. Examples: real baseline, warmup, targeted parallel A/B, disabling avoidable `empty_cache`, scheduler coefficient caching.
|
||||
- **P1:** meaningful code changes with contained risk. Examples: cross-attention KV caching, VAE gather/broadcast reduction, AdaLayerNorm/RMSNorm/RoPE fusion after trace evidence.
|
||||
- **P2:** high implementation or quality risk. Examples: FA to LA replacement, custom Triton/CUDA fused kernels, FP8/quantization, sparsity/Rainfusion-style acceleration.
|
||||
|
||||
Every implemented optimization needs A/B validation and a passing
|
||||
quality/precision gate, including math-preserving changes such as padding trim,
|
||||
layout cleanup, cache reuse, or host/runtime cleanup. Use objective metrics
|
||||
when available, keep reviewable artifacts, and compare against baseline
|
||||
self-run variance for precision, quantization, approximate kernels, or backend
|
||||
changes. If quality validation fails or is inconclusive, do not present the
|
||||
optimization as ready to merge.
|
||||
|
||||
## Optimization Layers
|
||||
|
||||
After baseline, parallel-search, and diagnostic traces, summarize optimization
|
||||
opportunities by layer. This is the core of the performance analysis: the goal
|
||||
is to connect evidence to a scoped implementation and a validation plan.
|
||||
|
||||
### Host and Runtime Optimization
|
||||
|
||||
Purpose: remove CPU/Python stalls, synchronization points, allocator overhead,
|
||||
and request-path overhead that leave GPU lanes empty.
|
||||
|
||||
Evidence to look for:
|
||||
|
||||
- High `idle_pct` or large `GAP` blocks in `trace_analyzer.py`.
|
||||
- Host-stack trace lines such as `torch.cuda.empty_cache`,
|
||||
`cudaStreamSynchronize`, `cudaDeviceSynchronize`, Python locks, scheduler
|
||||
waits, image/video preprocessing, or repeated small allocation paths.
|
||||
- Difference between client wall-clock time and server `inference_time_s`.
|
||||
|
||||
Typical candidates:
|
||||
|
||||
- Make avoidable `torch.cuda.empty_cache()` optional or guard it by memory
|
||||
headroom.
|
||||
- Cache scheduler coefficients, timesteps, masks, or other tiny repeated CPU
|
||||
computations when the request shape/steps are fixed.
|
||||
- Remove avoidable host-device synchronizations and blocking logging/stat calls.
|
||||
- Move expensive preprocessing out of the critical path or cache fixed prompt,
|
||||
image, and transform work for benchmark scenarios.
|
||||
- Ensure benchmark scripts record client-side elapsed time, HTTP status, output
|
||||
path, and server response metrics.
|
||||
|
||||
Priority guidance:
|
||||
|
||||
- Usually P0 when the change is measurement reliability or an obvious removable
|
||||
synchronization.
|
||||
- Usually P1 when it changes scheduling, memory lifetime, or request execution
|
||||
order.
|
||||
|
||||
Validation:
|
||||
|
||||
- Re-run non-profiler baseline with same workload and seed.
|
||||
- Confirm peak memory headroom if disabling cache cleanup.
|
||||
- Confirm generated output exists and quality/seed behavior is unchanged.
|
||||
|
||||
### Parallelism and Communication Optimization
|
||||
|
||||
Purpose: choose the right decomposition for CFG branches, sequence tokens, model
|
||||
weights, VAE tiles, and rank topology.
|
||||
|
||||
Evidence to look for:
|
||||
|
||||
- Baseline A/B across `CFG`, `USP/SP`, `Ring`, `HSDP/FSDP`, and VAE patch
|
||||
parallelism.
|
||||
- Stage timing shifts: `diffuse`, `vae.encode`, `vae.decode`, and server
|
||||
end-to-end.
|
||||
- NCCL kernel time from trace, not only `user_annotation` time.
|
||||
- Rank imbalance across SP group ranks or CFG branch ranks.
|
||||
- Memory headroom and OOM risk.
|
||||
|
||||
Typical candidates:
|
||||
|
||||
- `CFG=2 x USP=world/2` versus `CFG=1 x USP=world` for CFG-enabled models.
|
||||
- VAE patch parallel on/off or patch size tuning.
|
||||
- HSDP/FSDP on/off only if both configurations fit memory.
|
||||
- Ulysses versus Ulysses+Ring only after long-sequence `diffuse` is confirmed
|
||||
dominant and all-to-all is suspected.
|
||||
- Rank mapping/topology changes if all-rank traces show stragglers or NCCL wait.
|
||||
- Buffer reuse or preallocation for FSDP/HSDP all-gather paths.
|
||||
|
||||
Priority guidance:
|
||||
|
||||
- P0/P1 for configuration-only changes with strong measured wins.
|
||||
- P1 for buffer reuse or rank mapping changes.
|
||||
- P2 for invasive distributed algorithm changes.
|
||||
|
||||
Validation:
|
||||
|
||||
- Measure by stage and memory, not only end-to-end.
|
||||
- Use one-variable A/B with identical prompt/media/seed/shape/steps.
|
||||
- When communication is suspected, compare rank0-3 in one USP group and rank0
|
||||
versus rank4 across CFG branches for `CFG=2 x USP=4`.
|
||||
|
||||
### VAE and Media Pipeline Optimization
|
||||
|
||||
Purpose: reduce encode/decode, tiling, split/gather, and media conversion time.
|
||||
|
||||
Evidence to look for:
|
||||
|
||||
- Large `vae.encode` or `vae.decode` in low-overhead stage timings.
|
||||
- Host-stack gaps in VAE tile split, gather, merge, broadcast, or image/video
|
||||
transforms.
|
||||
- VAE kernels or cuDNN convolution in operator trace.
|
||||
- Whether every rank needs the final decoded tensor.
|
||||
|
||||
Typical candidates:
|
||||
|
||||
- Keep VAE patch parallel enabled when it has clear measured benefit.
|
||||
- Reduce VAE gather/broadcast to only ranks that need the final media output.
|
||||
- Reuse tile metadata, split buffers, or gather buffers.
|
||||
- Evaluate bf16/autocast behavior for VAE only with visual quality checks.
|
||||
- Avoid redundant image conversion, resize, or tensor construction in repeated
|
||||
benchmark runs.
|
||||
|
||||
Priority guidance:
|
||||
|
||||
- P0/P1 if VAE is a large share of the target workload or if a host gap is
|
||||
obvious and low risk.
|
||||
- Lower priority when `diffuse` dominates and VAE is already patch-parallelized.
|
||||
|
||||
Validation:
|
||||
|
||||
- Compare `vae.encode`, `vae.decode`, server end-to-end, and peak memory.
|
||||
- Check output video integrity, artifacts, flicker, and seed stability.
|
||||
|
||||
### Operator Fusion and Layout Cleanup
|
||||
|
||||
Purpose: reduce high-frequency small kernels, memory bandwidth pressure, layout
|
||||
conversions, and launch overhead in transformer and VAE blocks.
|
||||
|
||||
Evidence to look for:
|
||||
|
||||
- Top operator tables showing many `aten::copy_`, `aten::cat`,
|
||||
`split_with_sizes_copy`, `aten::add`, `aten::mul`, `aten::div`, norm,
|
||||
activation, RoPE, or reshape/layout kernels.
|
||||
- `ops_rankN.xlsx` `by_shape` sheet showing repeated small shapes inside the
|
||||
same block path.
|
||||
- Trace lanes showing many short kernels between larger GEMM/attention kernels.
|
||||
- Source code patterns with repeated elementwise chains or layout conversions.
|
||||
|
||||
Typical fusion targets:
|
||||
|
||||
- AdaLayerNorm / RMSNorm / LayerNorm plus scale/shift fusion.
|
||||
- RoPE fusion with Q/K layout preparation when shapes are stable.
|
||||
- Residual add, scale, gate, and elementwise chains.
|
||||
- MLP gate/up/down cleanup, such as fusing activation and multiply around
|
||||
GEMM outputs when feasible.
|
||||
- QKV projection and reshape/split/cat path cleanup.
|
||||
- Attention pre/post layout cleanup to avoid unnecessary copies, cats, and
|
||||
splits around sequence parallel all-to-all.
|
||||
|
||||
Priority guidance:
|
||||
|
||||
- P1 when implemented with existing PyTorch/Triton/local helper patterns and
|
||||
validated against exact outputs or tolerances.
|
||||
- P2 when it requires custom CUDA/Triton kernels, changes numerics, or touches
|
||||
attention math directly.
|
||||
|
||||
Validation:
|
||||
|
||||
- First prove the operator family is material for the target shape.
|
||||
- Use non-profiler A/B for latency and stage timing.
|
||||
- Use quality regression checks for generated video stability.
|
||||
- Check compile behavior and graph breaks if using `torch.compile`.
|
||||
|
||||
### Attention Main-Path Optimization
|
||||
|
||||
Purpose: address the dominant self-attention cost when FlashAttention or other
|
||||
attention kernels dominate CUDA time.
|
||||
|
||||
Evidence to look for:
|
||||
|
||||
- Operator trace where FlashAttention/SDPA kernels dominate total CUDA time.
|
||||
- Attention shape from `ops_rankN.xlsx` `by_shape`, model code, or trace
|
||||
metadata.
|
||||
- Whether attention cost scales with latent frames, latent H/W, patch size, or
|
||||
CFG duplication.
|
||||
- Layout/copy/all-to-all work around attention.
|
||||
|
||||
Typical candidates:
|
||||
|
||||
- Verify the attention backend and shape are on the intended fast kernel path.
|
||||
- Compare supported attention backends only with identical workload and quality
|
||||
settings.
|
||||
- Reduce attention input size by safe model/config choices when allowed:
|
||||
latent resolution, frame count, patching, boundary ratio, or windowing.
|
||||
- Remove avoidable layout conversions before/after attention.
|
||||
- Reuse condition-side KV or other static inputs if the model structure allows.
|
||||
- Consider custom kernels, sparse/window/linear attention, or approximation only
|
||||
after quality risk is accepted.
|
||||
|
||||
Priority guidance:
|
||||
|
||||
- P1 for backend/config/layout changes with preserved math.
|
||||
- P2 for approximate attention, sparsity, custom kernels, or any change that can
|
||||
alter quality/temporal consistency.
|
||||
|
||||
Validation:
|
||||
|
||||
- Always include output quality and seed behavior checks.
|
||||
- Compare `diffuse`, server end-to-end, peak memory, and attention kernel time
|
||||
in diagnostic traces if needed.
|
||||
|
||||
### Algorithmic, Precision, and Approximation Optimization
|
||||
|
||||
Purpose: reduce mathematical work or precision cost beyond local code cleanup.
|
||||
|
||||
Evidence to look for:
|
||||
|
||||
- A single operator family dominates even after low-risk runtime, parallel, and
|
||||
fusion work.
|
||||
- Memory bandwidth or compute utilization suggests precision or quantization
|
||||
could matter.
|
||||
- The user explicitly allows quality-preserving or approximate methods.
|
||||
|
||||
Typical candidates:
|
||||
|
||||
- FP8/quantization for transformer or selected projections.
|
||||
- Sparsity or Rainfusion-style acceleration.
|
||||
- Reduced steps, scheduler changes, distillation, or caching across frames.
|
||||
- Approximate attention or linear attention.
|
||||
|
||||
Priority guidance:
|
||||
|
||||
- Usually P2 because quality, numerics, and implementation risk are high.
|
||||
|
||||
Validation:
|
||||
|
||||
- Requires strict A/B, visual quality review, temporal flicker checks, seed
|
||||
stability, and possibly human evaluation.
|
||||
|
||||
### Interpolation, Super-Resolution, and E2E Pipeline Optimization
|
||||
|
||||
Purpose: optimize the whole user-visible video product, not only the base
|
||||
diffusion invocation. Some deployments trade base-model latency against
|
||||
post-processing, interpolation, or super-resolution stages.
|
||||
|
||||
Evidence to look for:
|
||||
|
||||
- E2E latency breakdown across base generation, interpolation, super-resolution,
|
||||
encoding, storage, and response streaming.
|
||||
- Fast/slow GPU or fast/slow stage analysis across multiple cards and pipeline
|
||||
stages.
|
||||
- User quality target: resolution, FPS, temporal smoothness, and acceptable
|
||||
post-processing artifacts.
|
||||
|
||||
Typical candidates:
|
||||
|
||||
- Add or optimize a frame interpolation stage when it reduces required base
|
||||
model frames for the same perceived FPS.
|
||||
- Add or optimize a super-resolution model when generating lower base
|
||||
resolution plus SR is faster for the target quality.
|
||||
- Analyze E2E2 pipeline behavior: client request, service scheduling, diffusion,
|
||||
VAE/media, post-process, file write, and response.
|
||||
- Identify fast/slow cards or stages and rebalance pipeline placement.
|
||||
|
||||
Priority guidance:
|
||||
|
||||
- P1 when using proven interpolation/SR components without changing diffusion
|
||||
math.
|
||||
- P2 when quality risk is high or the pipeline adds significant operational
|
||||
complexity.
|
||||
|
||||
Validation:
|
||||
|
||||
- Measure E2E wall-clock, per-stage server timings, output FPS/resolution,
|
||||
artifacts, flicker, and user-visible quality.
|
||||
|
||||
### Optimization Candidate Library
|
||||
|
||||
Use this table as a compact menu, not as automatic recommendations. Pick items
|
||||
only when baseline metrics, trace evidence, source inspection, or quality
|
||||
tolerance supports them.
|
||||
|
||||
| Layer | Candidate | Evidence | Priority | Validation focus |
|
||||
|---|---|---|---|---|
|
||||
| Measurement | Freeze server/request/benchmark commands | Missing or drifting commands | P0 | Repeatable non-profiler A/B |
|
||||
| Measurement | Separate baseline and diagnostic profiler runs | Profiler used for latency claims | P0 | Low-overhead stage timings |
|
||||
| Host/runtime | Guard or remove avoidable `empty_cache` | Host-stack gaps or sync stalls | P0/P1 | Latency, peak memory, OOM safety |
|
||||
| Host/runtime | Cache scheduler coefficients/timesteps | Repeated tiny CPU/GPU work | P0/P1 | Same seed/output, stage timing |
|
||||
| Host/runtime | Reduce framework scheduling overhead | Client time exceeds server time | P1 | E2E latency, throughput |
|
||||
| Parallel | `CFG=2 x USP=world/2` vs `USP=world` | CFG doubles forward work | P0/P1 | `diffuse`, NCCL, memory |
|
||||
| Parallel | Tune VAE patch parallelism | VAE encode/decode is material | P0/P1 | VAE time, output correctness |
|
||||
| Parallel | HSDP on/off or buffer reuse | HSDP affects memory/all-gather | P1 | Memory, latency, OOM risk |
|
||||
| Parallel | Ulysses vs Ulysses+Ring | Long sequence all-to-all suspected | P1/P2 | Rank balance, NCCL kernels |
|
||||
| Cross-attn | Disable SP for short condition tokens | Cross-attn comm exceeds compute | P1 | `diffuse`, correctness |
|
||||
| VAE/media | Reduce VAE gather/broadcast | Rank traces show VAE wait | P1 | Rank balance, output file |
|
||||
| VAE/media | Reuse tile metadata/buffers | Tile split/merge host gaps | P1 | `vae.encode/decode`, memory |
|
||||
| VAE/media | VAE bf16/autocast | VAE float kernels are slow | P1/P2 | Artifacts, flicker, seed stability |
|
||||
| Operator fusion | AdaLayerNorm/LayerNorm fusion | Norm plus scale/shift kernels | P1 | Numeric tolerance, latency |
|
||||
| Operator fusion | RMSNorm fusion | Many small RMSNorm kernels | P1 | Numeric tolerance, latency |
|
||||
| Operator fusion | RoPE cache/fuse/layout cleanup | RoPE copy/reshape kernels | P1/P2 | Kernel count, correctness |
|
||||
| Operator layout | QKV or attention layout cleanup | Copy/cat/split around attention | P1 | Copy kernels, compile behavior |
|
||||
| Attention | Verify backend fast path | FA/SDPA dominates trace | P1 | `diffuse`, attention kernels |
|
||||
| Attention | FA to LA or selected-head LA | Attention remains dominant | P2 | Quality, temporal stability |
|
||||
| Precision | Transformer FP8/quantization | Compute/bandwidth bound and allowed | P2 | Quality, speed, stability |
|
||||
| Sparsity | Rainfusion-style acceleration | DiT compute remains dominant | P2 | Prompt diversity, quality |
|
||||
| Pipeline | Frame interpolation | Fewer base frames can meet FPS | P1/P2 | E2E latency, motion artifacts |
|
||||
| Pipeline | Super-resolution | Lower base res plus SR may win | P1/P2 | Detail quality, artifacts |
|
||||
| E2E | Fast/slow-card analysis | Multi-card stragglers | P0/P1 | Per-rank/stage wall-clock |
|
||||
|
||||
### Optimization Plan Template
|
||||
|
||||
Use this table shape when reporting the next work items:
|
||||
|
||||
| Priority | Layer | Candidate | Evidence | Expected benefit | Implementation path | Validation | Quality risk |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| P0 | Host/runtime | Guard `empty_cache` | Host-stack gap points to `torch.cuda.empty_cache` | Small latency reduction, less idle | Add config/env guard | Non-profiler A/B, memory check | Low |
|
||||
| P1 | Operator fusion | RMSNorm/AdaLayerNorm fusion | High-frequency norm/elementwise kernels | Lower launch/bandwidth overhead | Use existing fusion helper or targeted Triton | A/B + output check | Medium |
|
||||
| P1/P2 | Attention | Attention layout/backend investigation | FA kernel dominates CUDA time | Potentially large | Inspect shapes/backend and remove layout copies | A/B + trace + quality | Medium/high |
|
||||
|
||||
Then present a short selection prompt using the same rows:
|
||||
|
||||
```text
|
||||
Which candidate should we implement next?
|
||||
|
||||
1. P0 Host/runtime: guard empty_cache
|
||||
- Expected benefit: small but low-risk latency reduction.
|
||||
- Risk: possible memory increase/OOM if memory headroom is insufficient.
|
||||
|
||||
2. P1 Operator fusion: inspect by_shape and implement first norm/RoPE/layout fusion
|
||||
- Expected benefit: medium if high-frequency small kernels are confirmed.
|
||||
- Risk: numerical/compile/quality validation needed.
|
||||
|
||||
3. P1/P2 Attention: FA/LA/backend/layout investigation
|
||||
- Expected benefit: potentially large.
|
||||
- Risk: high quality and implementation risk.
|
||||
```
|
||||
|
||||
If the user has not chosen an item, default to explaining tradeoffs and asking
|
||||
which candidate to execute. Only proceed autonomously on low-risk P0 measurement
|
||||
or instrumentation fixes.
|
||||
|
||||
## Analysis Helpers
|
||||
|
||||
PyTorch profiler traces are Chrome/Perfetto-compatible JSON files, usually
|
||||
`trace_rankN.json` or `trace_rankN.json.gz`. They normally contain a top-level
|
||||
`traceEvents` list, though some exporters emit the raw event list directly.
|
||||
|
||||
Use the checked-in analyzer from the repository root:
|
||||
|
||||
```bash
|
||||
python3 .claude/skills/diffusion-perf-opt/scripts/trace_analyzer.py \
|
||||
vllm_profile/.../trace_rank0.json.gz \
|
||||
--min-gap-ms 5 \
|
||||
--topn 20
|
||||
```
|
||||
|
||||
For rank imbalance or communication questions, pass all relevant rank traces in
|
||||
one command. For host gaps, lower `--min-gap-ms` to `1` and use a host-stack
|
||||
trace. Read `gpu_span_s`, `busy_union_s`, `idle_union_s`, `idle_pct`, `GAP`
|
||||
blocks, `Top GPU/operator events by total duration`, and `Top NCCL-like events
|
||||
by category`. Treat `cat=user_annotation` NCCL ranges as enclosing annotations;
|
||||
prefer `cat=kernel` or `cat=gpu_user_annotation` for real device work.
|
||||
|
||||
The analyzer summarizes timing only. It does not parse tensor shapes, attribute
|
||||
overlap to individual streams, prove quality, or provide final latency claims.
|
||||
Use `ops_rankN.xlsx` or PyTorch key averages for shape analysis, and re-test any
|
||||
optimization with non-profiler baseline commands.
|
||||
|
||||
Read `references/optimization-playbook.md` when drafting the optimization table or comparing candidate techniques.
|
||||
|
||||
## vLLM Omni Heuristics
|
||||
|
||||
- Cross-attention usually should not use USP/SP when text/image condition token count is much smaller than latent video tokens. Confirm via trace; in Wan2.2 I2V, self-attention dominates cross-attention.
|
||||
- VAE bf16/autocast is often worthwhile but requires visual quality checks.
|
||||
- VAE patch parallel can help decode/encode but may add gather/merge/broadcast overhead. Check whether all ranks need the final decoded tensor.
|
||||
- HSDP/FSDP is primarily a memory strategy. If the model fits without it, run an on/off latency comparison.
|
||||
- Scheduler work can create small host/device gaps; cache tiny solve coefficients when timesteps/order are known.
|
||||
- `torch.cuda.empty_cache()` can prevent OOM but creates synchronization/idle. Make it optional if memory headroom is sufficient.
|
||||
- `Command Buffer Full` in profiler output is profiler overhead, not a model optimization target.
|
||||
@@ -1,4 +0,0 @@
|
||||
interface:
|
||||
display_name: "vLLM Omni Perf Opt"
|
||||
short_description: "Profile and optimize vLLM Omni diffusion performance."
|
||||
default_prompt: "Analyze and optimize a vLLM Omni diffusion workload using baselines, traces, parallel strategy modeling, and prioritized optimization plans."
|
||||
@@ -1,82 +0,0 @@
|
||||
# Optimization Playbook
|
||||
|
||||
Use this reference when turning trace evidence into prioritized vLLM Omni optimizations.
|
||||
|
||||
## Parallel Strategy
|
||||
|
||||
Start from workload shape:
|
||||
|
||||
- Long video tokens, high resolution, many frames: self-attention and FFN dominate. USP/SP is usually valuable.
|
||||
- CFG enabled with positive and negative prompts: transformer forward count doubles. CFG parallel may beat USP on small/moderate sequence lengths.
|
||||
- Cross-attention: skip SP/USP when condition tokens are much smaller than latent tokens; extra all-to-all can exceed compute saved.
|
||||
- HSDP/FSDP: use to fit memory first. If memory allows, compare without HSDP because all-gathers and lazy init add overhead.
|
||||
- VAE parallel: useful when encode/decode is material. Check gather/merge/broadcast and whether every rank needs final decoded output.
|
||||
|
||||
Candidate matrices:
|
||||
|
||||
- 2 cards: `USP=2`, `CFG=2`, HSDP on/off, VAE PP on/off.
|
||||
- 4 cards: `USP=4`, `CFG=2 x USP=2`, `USP=2 x HSDP`, VAE PP on/off.
|
||||
- 8 cards: add pipeline/stage parallel only after measuring orchestration and inter-stage transfer overhead.
|
||||
|
||||
## Host And Free Bubble Analysis
|
||||
|
||||
Look for device intervals where no kernel, memcpy, or memset is active.
|
||||
|
||||
Common causes:
|
||||
|
||||
- First request FSDP/HSDP lazy init.
|
||||
- Python preprocessing: image load, resize, `to_tensor`, tokenizer, request marshaling.
|
||||
- `torch.cuda.empty_cache()` or explicit synchronizations.
|
||||
- Scheduler small linear algebra such as tiny `torch.linalg.solve`.
|
||||
- VAE tile split/merge Python loops.
|
||||
- Profiler overhead such as CUPTI command buffer flush/full.
|
||||
|
||||
Also inspect trace-external time:
|
||||
|
||||
- API request parsing and multipart upload.
|
||||
- Engine/orchestrator dispatch.
|
||||
- Worker RPC and queue wait.
|
||||
- Output serialization, video write, base64, HTTP response.
|
||||
|
||||
## Operator Optimization Candidates
|
||||
|
||||
Evaluate necessity before implementation:
|
||||
|
||||
- **FA to LA / LA for head:** high potential only if attention is the main CUDA hotspot and target shapes fit LA well.
|
||||
- **Cross-attention BSND/no transpose:** do only if transpose/copy around cross-attn is measurable.
|
||||
- **AdaLayerNorm fusion:** fuse norm, scale/shift, gate, residual, and dtype conversion if repeated elementwise kernels are significant.
|
||||
- **RMSNorm fusion:** useful for Q/K norms and VAE norms when small-kernel launch count is high.
|
||||
- **FP32 LayerNorm to bf16:** quality-sensitive; run visual regression.
|
||||
- **LA preprocess:** integrate only if preprocess overhead is cached or amortized.
|
||||
- **RoPE move/cache/fuse:** useful when RoPE small kernels are visible around every self-attn block.
|
||||
- **VAE RMSNorm replacement:** only after VAE internal trace shows norm as a real component of decode/encode.
|
||||
- **Scheduler fusion/cache:** cache coefficient solves and fuse CFG combine with scheduler arithmetic when possible.
|
||||
|
||||
Before writing a custom kernel, check:
|
||||
|
||||
- Local vLLM Omni implementations.
|
||||
- Upstream vLLM kernels and attention backends.
|
||||
- SGLang, FlashInfer, FlashAttention, and Triton examples.
|
||||
- Shape constraints and layout expectations.
|
||||
|
||||
## Quantization And Sparsity
|
||||
|
||||
Treat these as high-risk optimizations:
|
||||
|
||||
- Rainfusion/sparse diffusion can be high reward but needs model-specific quality validation.
|
||||
- VAE decode quantization can reduce decode time but usually has smaller end-to-end upside than transformer attention/FFN work.
|
||||
- FP8 or approximate kernels require seed-stable visual regression, temporal flicker checks, and prompt diversity.
|
||||
|
||||
## Output Format For Recommendations
|
||||
|
||||
Return a table with:
|
||||
|
||||
- Priority: P0/P1/P2.
|
||||
- Optimization.
|
||||
- Evidence from trace or code.
|
||||
- Expected benefit.
|
||||
- Risk/quality impact.
|
||||
- Implementation path.
|
||||
- Validation command or A/B test.
|
||||
|
||||
State clearly when a recommendation is a hypothesis requiring new trace with shapes.
|
||||
@@ -1,144 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize torch profiler Chrome traces for vLLM Omni diffusion optimization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
GPU_CATS = {"kernel", "gpu_memcpy", "gpu_memset"}
|
||||
CPU_CATS = {"python_function", "user_annotation", "cpu_op", "cuda_runtime", "cuda_driver"}
|
||||
|
||||
|
||||
def open_trace(path: Path):
|
||||
if path.suffix == ".gz":
|
||||
return gzip.open(path, "rt")
|
||||
return path.open("rt")
|
||||
|
||||
|
||||
def event_name(event: dict[str, Any]) -> str:
|
||||
return str(event.get("name", ""))
|
||||
|
||||
|
||||
def summarize_trace(path: Path, min_gap_us: float, topn: int) -> None:
|
||||
with open_trace(path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
events = data if isinstance(data, list) else data.get("traceEvents", [])
|
||||
gpu: list[tuple[float, float, float, str, str, Any, Any]] = []
|
||||
cpu: list[tuple[float, float, float, str, str, Any, Any]] = []
|
||||
by_gpu_name: dict[str, list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
|
||||
by_name: dict[str, list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
|
||||
nccl: dict[tuple[str, str], list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
|
||||
|
||||
for event in events:
|
||||
dur = event.get("dur")
|
||||
ts = event.get("ts")
|
||||
if dur is None or ts is None or dur <= 0:
|
||||
continue
|
||||
cat = str(event.get("cat", ""))
|
||||
name = event_name(event)
|
||||
row = (float(ts), float(ts + dur), float(dur), name, cat, event.get("pid"), event.get("tid"))
|
||||
stat = by_name[name]
|
||||
stat[0] += 1
|
||||
stat[1] += dur
|
||||
stat[2] = max(stat[2], dur)
|
||||
if cat in GPU_CATS:
|
||||
gpu.append(row)
|
||||
gstat = by_gpu_name[name]
|
||||
gstat[0] += 1
|
||||
gstat[1] += dur
|
||||
gstat[2] = max(gstat[2], dur)
|
||||
elif cat in CPU_CATS:
|
||||
cpu.append(row)
|
||||
if "nccl" in name.lower():
|
||||
nstat = nccl[(cat, name)]
|
||||
nstat[0] += 1
|
||||
nstat[1] += dur
|
||||
nstat[2] = max(nstat[2], dur)
|
||||
|
||||
print(f"\n== {path}")
|
||||
print(f"events={len(events)} gpu_events={len(gpu)} cpu_events={len(cpu)}")
|
||||
if not gpu:
|
||||
print("No GPU events found.")
|
||||
return
|
||||
|
||||
gpu.sort()
|
||||
merged: list[list[Any]] = []
|
||||
for start, end, dur, name, cat, pid, tid in gpu:
|
||||
if not merged or start > merged[-1][1]:
|
||||
merged.append([start, end, [(start, end, dur, name, cat, pid, tid)]])
|
||||
else:
|
||||
merged[-1][1] = max(merged[-1][1], end)
|
||||
merged[-1][2].append((start, end, dur, name, cat, pid, tid))
|
||||
|
||||
span = max(end for _, end, *_ in gpu) - min(start for start, *_ in gpu)
|
||||
busy = sum(end - start for start, end, _ in merged)
|
||||
idle = span - busy
|
||||
print(
|
||||
f"gpu_span_s={span / 1e6:.3f} "
|
||||
f"busy_union_s={busy / 1e6:.3f} "
|
||||
f"idle_union_s={idle / 1e6:.3f} "
|
||||
f"idle_pct={idle / span * 100:.2f}"
|
||||
)
|
||||
|
||||
interesting_cpu = [
|
||||
row
|
||||
for row in cpu
|
||||
if row[2] >= 1000
|
||||
and (
|
||||
row[4] in {"python_function", "user_annotation"}
|
||||
or "cudaStreamSynchronize" in row[3]
|
||||
or "cudaDeviceSynchronize" in row[3]
|
||||
or "cudaLaunch" in row[3]
|
||||
or "cudaMemcpy" in row[3]
|
||||
)
|
||||
]
|
||||
|
||||
gaps = []
|
||||
for idx in range(1, len(merged)):
|
||||
gap_start = merged[idx - 1][1]
|
||||
gap_end = merged[idx][0]
|
||||
gap_dur = gap_end - gap_start
|
||||
if gap_dur >= min_gap_us:
|
||||
prev_event = max(merged[idx - 1][2], key=lambda x: x[1])
|
||||
next_event = min(merged[idx][2], key=lambda x: x[0])
|
||||
mid = (gap_start + gap_end) / 2
|
||||
containers = [row for row in interesting_cpu if row[0] <= mid <= row[1]]
|
||||
containers = sorted(containers, key=lambda x: x[2])[:8]
|
||||
gaps.append((gap_dur, gap_start, gap_end, prev_event, next_event, containers))
|
||||
|
||||
print(f"gaps_ge_{min_gap_us / 1000:.3f}ms count={len(gaps)} sum_s={sum(g[0] for g in gaps) / 1e6:.3f}")
|
||||
for gap_dur, gap_start, gap_end, prev_event, next_event, containers in sorted(gaps, reverse=True)[:topn]:
|
||||
print(f"\nGAP {gap_dur / 1000:.3f} ms ts={gap_start:.0f}->{gap_end:.0f}")
|
||||
print(f" prev {prev_event[4]} {prev_event[2] / 1000:.3f} ms {prev_event[3][:160]}")
|
||||
print(f" next {next_event[4]} {next_event[2] / 1000:.3f} ms {next_event[3][:160]}")
|
||||
for row in containers:
|
||||
print(f" in {row[4]} {row[2] / 1000:.3f} ms {row[3][:180]}")
|
||||
|
||||
print("\nTop GPU/operator events by total duration:")
|
||||
for name, (count, total, max_dur) in sorted(by_gpu_name.items(), key=lambda kv: kv[1][1], reverse=True)[:topn]:
|
||||
print(f" {int(count):8d} total={total / 1e6:9.3f}s max={max_dur / 1000:9.3f}ms {name[:180]}")
|
||||
|
||||
print("\nTop NCCL-like events by category:")
|
||||
for (cat, name), (count, total, max_dur) in sorted(nccl.items(), key=lambda kv: kv[1][1], reverse=True)[:topn]:
|
||||
print(f" {int(count):8d} total={total / 1e6:9.3f}s max={max_dur / 1000:9.3f}ms cat={cat} {name[:160]}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("traces", nargs="+", type=Path, help="trace.json or trace.json.gz files")
|
||||
parser.add_argument("--min-gap-ms", type=float, default=5.0, help="minimum GPU idle gap to print")
|
||||
parser.add_argument("--topn", type=int, default=20, help="number of gaps/hotspots to print")
|
||||
args = parser.parse_args()
|
||||
|
||||
for trace in args.traces:
|
||||
summarize_trace(trace, args.min_gap_ms * 1000.0, args.topn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,135 +0,0 @@
|
||||
---
|
||||
name: find-simplifications
|
||||
description: Find evidence-backed simplification candidates in vLLM-Omni and, when requested, turn them into focused proposals or code changes. Use for audits of dead, duplicated, speculative, over-generalized, unnecessarily defensive, or hand-rolled code. Use review-pr for ordinary correctness review and diffusion-perf-opt for performance-first optimization.
|
||||
---
|
||||
|
||||
# Find vLLM-Omni Simplifications
|
||||
|
||||
Find a small number of well-proven ways to reduce code, state, APIs, or maintenance cost without weakening supported behavior. Treat this as an architecture and ownership audit, not a dead-code search or style cleanup.
|
||||
|
||||
The default deliverable is a read-only report. Do not add TODOs, design documents, issues, or code unless the user asks for those changes.
|
||||
|
||||
## Establish the Contract
|
||||
|
||||
Before judging a candidate:
|
||||
|
||||
1. Read the repository instructions supplied by the host.
|
||||
2. Read [`docs/design/architecture_overview.md`](../../../docs/design/architecture_overview.md).
|
||||
3. Select only the relevant active module and feature documents from [`docs/design/index.md`](../../../docs/design/index.md). Archived design pages are historical context, not current contracts.
|
||||
4. Identify the comparison base with `git merge-base HEAD origin/main`; do not assume the current branch represents the intended baseline.
|
||||
5. If tests or CI coverage may change, use [`vllm-omni-test`](../vllm-omni-test/SKILL.md) for test placement, markers, and commands.
|
||||
|
||||
Separate an implementation accident from an intentional boundary. In particular:
|
||||
|
||||
- vLLM-Omni extends and sometimes overrides upstream vLLM. A similar upstream implementation is evidence for reuse only after comparing API, lifecycle, and multi-stage semantics.
|
||||
- Registries, import strings, model configuration, deployment YAML, plugins, and optional dependencies create dynamic consumers that a static reference count can miss.
|
||||
- Hardware backends, model families, execution modes, and connector implementations may be intentionally parallel. Do not propose collapsing one only because it is inactive in the local environment.
|
||||
- Stage, replica, request, cache, and transport state can look redundant while representing different owners. Prove that two fields or barriers encode the same fact before merging them.
|
||||
- A shorter hot path is not a simplification if it adds synchronization, tensor copies, host materialization, recompilation, or a measurable quality/performance regression.
|
||||
|
||||
## Strong Candidates
|
||||
|
||||
A strong candidate removes a meaningful surface and has a clear replacement or deletion boundary. Examples include:
|
||||
|
||||
- A public method, configuration option, registry entry, route, message field, or event has no production consumer and no compatibility obligation.
|
||||
- Tests or documentation are the only consumers of behavior that is not part of a supported contract.
|
||||
- Multiple layers mirror the same request, lifecycle, cache-validity, readiness, or topology state.
|
||||
- AR, diffusion, platform, or model-specific implementations duplicate a stable common operation and can share it without hiding performance-critical differences.
|
||||
- An inherited upstream vLLM primitive is reimplemented locally without an Omni-specific semantic difference.
|
||||
- Compatibility or fallback branches protect environments the project no longer supports.
|
||||
- Serialization, IPC, output formatting, or connector code materializes the same payload more than once or converts between equivalent representations.
|
||||
- A helper, abstraction, package split, or extension point has only one real implementation and adds more indirection than policy.
|
||||
- Hand-written infrastructure can be replaced by an existing project dependency, Python standard-library feature, PyTorch primitive, or upstream vLLM facility with net deletion and equal observability.
|
||||
- Tests maintain large mocks, snapshots, or expected data solely for an unused API or representation.
|
||||
|
||||
Do not elevate typo fixes, formatting, isolated renames, or vague complexity complaints into simplification candidates. Bundle small cleanups only when they support one substantive removal.
|
||||
|
||||
## Survey by Ownership Boundary
|
||||
|
||||
For broad audits, cover the largest or riskiest production deltas first. Useful domains are:
|
||||
|
||||
- `vllm_omni/engine/`: orchestration, stage pools, request state, output convergence, cancellation, and replica lifecycle.
|
||||
- `vllm_omni/worker/` and AR model execution: upstream vLLM overlays, scheduler/cache state, collective RPC, adapters, and weight loading.
|
||||
- `vllm_omni/diffusion/`: request/step execution, batching, output materialization, IPC, offload, parallelism, and model pipelines.
|
||||
- `vllm_omni/entrypoints/`: duplicated validation/defaulting, offline/online parity, streaming, and route compatibility.
|
||||
- Connectors and distributed code: ownership, copies, barriers, handles, cleanup, and failure propagation across process/device/node boundaries.
|
||||
- Platform backends: genuine vendor differences versus copied control flow that has drifted.
|
||||
- Configuration, registries, recipes, and examples: duplicate sources of truth and options that never reach a runtime consumer.
|
||||
- Tests and CI: redundant fixtures, mocks of obsolete contracts, and expensive coverage that can move to a cheaper level without losing the guarded behavior.
|
||||
|
||||
When the user explicitly requests a repository-wide or many-candidate audit and parallel agent work is available and authorized, divide these domains among agents and require the same evidence format from each. Otherwise survey them sequentially.
|
||||
|
||||
## Prove or Reject Each Candidate
|
||||
|
||||
Use `rg` first, then read every relevant caller and implementation. Search exact Python symbols plus serialized names, route paths, configuration keys, registry strings, YAML values, and CLI spellings.
|
||||
|
||||
Classify evidence:
|
||||
|
||||
- **Production:** `vllm_omni/`, runtime configuration, registries, serving entrypoints, connectors, recipes used for supported deployments, and executable loader paths.
|
||||
- **Validation/support:** `tests/`, benchmarks, examples, docs, CI, and developer tooling.
|
||||
- **External/dynamic:** upstream vLLM contracts, third-party plugins, model repositories, optional backends, and string-loaded objects. State what was checked and what remains unknown.
|
||||
|
||||
For each candidate, answer:
|
||||
|
||||
1. What exact surface would disappear or merge?
|
||||
2. Who owns the state or behavior today?
|
||||
3. Which production consumers exist, including dynamic consumers?
|
||||
4. What supported behavior, compatibility, performance, memory, or observability could change?
|
||||
5. What is the smallest validation that would catch a bad simplification?
|
||||
6. Is the result net deletion after tests, adapters, migration glue, and documentation are counted?
|
||||
|
||||
Reject or downgrade the candidate when a live consumer exists and removal is actually a feature decision; when an active design contract explains the separation; when the change only moves complexity; or when hardware/model evidence is unavailable for a performance-sensitive claim.
|
||||
|
||||
## Lifecycle and Data-Movement Audits
|
||||
|
||||
For asynchronous or distributed code, map the ownership graph before recommending deletion:
|
||||
|
||||
```text
|
||||
request admission -> orchestrator -> stage/replica -> worker/device
|
||||
-> connector or IPC boundary -> output aggregation -> terminal cleanup
|
||||
```
|
||||
|
||||
Associate every lock, event, sentinel, queue, readiness flag, timeout, abort path, and cache marker with an owner and transition. Coalesce mechanisms only when they represent the same state and have the same failure boundary. Preserve separate mechanisms when they protect publication versus rollback, local versus remote ownership, first-terminal-outcome arbitration, or cleanup after partial failure.
|
||||
|
||||
For tensors and media, also trace device, process, and ownership transitions. Count GPU-to-CPU copies, shared-memory/object-store handles, serialization, decode/encode steps, and unlink/release responsibility. Prefer eliminating a transfer or representation over merely hiding it behind a helper.
|
||||
|
||||
## Dependency Replacements
|
||||
|
||||
Prefer facilities already available in the repository before adding a dependency. For a proposed replacement:
|
||||
|
||||
- Name the exact local behavior it covers and any semantics that remain in glue code.
|
||||
- Check maintenance, license, optional-dependency boundaries, platform support, and transitive footprint.
|
||||
- Compare net deletion, test burden, import/startup cost, and runtime behavior.
|
||||
- Benchmark when the replaced code is on a model, scheduler, connector, serialization, or serving hot path.
|
||||
|
||||
A wrapper that retains the same state machine or conversion logic is not a simplification.
|
||||
|
||||
## Deliver the Result
|
||||
|
||||
Report only candidates supported by concrete evidence. For each one include:
|
||||
|
||||
- **Surface:** files and symbols involved.
|
||||
- **Evidence:** production consumers found, dynamic paths checked, and relevant design contract.
|
||||
- **Simplification:** exactly what to remove, merge, inline, or reuse.
|
||||
- **Benefit:** deleted API/state/code, reduced data movement, or reduced maintenance/test burden.
|
||||
- **Risk and validation:** behavior that could change and the narrowest useful tests or measurements.
|
||||
- **Confidence:** high or medium, with the unresolved evidence named for medium-confidence items.
|
||||
|
||||
Also list representative ideas that were rejected when that prevents repeated investigation. Prefer three high-confidence candidates over a long speculative inventory.
|
||||
|
||||
If the user asks for a durable design proposal, place it under `docs/design/feature/` and register it in `docs/design/index.md`; use current design-document conventions and keep implementation details proportional to the decision. If the user asks for implementation, make the smallest coherent change, update affected tests/docs, and preserve public compatibility unless the user explicitly accepts a breaking change.
|
||||
|
||||
## Folding Work from Another Branch
|
||||
|
||||
Diff the source branch against its own merge base with `origin/main`, not against the current feature branch. Port only independently justified candidates. Consolidate overlaps into the candidate with stronger evidence, and do not preserve a candidate count by carrying weaker duplicates.
|
||||
|
||||
## Validation
|
||||
|
||||
Match validation to the outgoing diff:
|
||||
|
||||
- Documentation or skill changes: run the repository skill validator when available, check links/paths, search for stale source-specific terminology, and run `git diff --check`.
|
||||
- Python changes: run the narrowest relevant pytest coverage through the repository environment, then lint/format the changed files.
|
||||
- Distributed, model, hardware, performance, or accuracy changes: run available CPU/static checks and state the exact GPU/NPU/model gap; never present simulated evidence as device validation.
|
||||
- Before submission, use [`precheck-pr`](../precheck-pr/SKILL.md) for the branch-level self-review.
|
||||
|
||||
When summarizing, distinguish checks that passed from checks that were not runnable, and state the snapshot or branch against which the audit was performed.
|
||||
@@ -1,108 +0,0 @@
|
||||
---
|
||||
name: precheck-pr
|
||||
description: Self-check your branch before creating a PR — catch dead code, prevent new model-specific Python examples, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
|
||||
---
|
||||
|
||||
# PR Pre-Check
|
||||
|
||||
Self-review your branch before creating a PR against `vllm-project/vllm-omni`. Two modes: **quick** catches showstoppers, **full** does a thorough maintainer-grade review. Never posts to GitHub; the report is for the contributor's terminal only.
|
||||
|
||||
## Mode Selection
|
||||
|
||||
| Mode | When | Time |
|
||||
|------|------|------|
|
||||
| **Quick** | About to push, final sanity check | ~3 min |
|
||||
| **Full** | Ready for review, want maintainer-level scan | ~10 min |
|
||||
|
||||
Default to quick if unsure. Run full before marking a PR "ready for review."
|
||||
|
||||
## Workflow
|
||||
|
||||
### Step 1: Detect Base Branch
|
||||
|
||||
```bash
|
||||
BASE_SHA=$(git merge-base HEAD origin/main 2>/dev/null \
|
||||
|| git merge-base HEAD main 2>/dev/null \
|
||||
|| echo origin/main)
|
||||
echo "diffing against ${BASE_SHA}"
|
||||
git diff --name-only ${BASE_SHA}...HEAD
|
||||
```
|
||||
|
||||
### Step 2: Validate PR Title
|
||||
|
||||
Check the most recent commit message (or branch name if no commit yet) against the project convention. Valid prefixes:
|
||||
|
||||
| Prefix | Applies to |
|
||||
|--------|-----------|
|
||||
| `[Bugfix]` | Bug fixes |
|
||||
| `[CI/Build]` | Build or CI improvements |
|
||||
| `[Doc]` | Documentation changes |
|
||||
| `[Model]` | New/improved models (include model name) |
|
||||
| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
|
||||
| `[Kernel]` | CUDA/kernel changes |
|
||||
| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
|
||||
| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
|
||||
| `[Misc]` | Other changes (use sparingly) |
|
||||
|
||||
✗ if: missing prefix, wrong case (`[bugfix]`), or WIP/Draft in title.
|
||||
⚠ if: `[Model]` prefix without the model identifier (e.g., `[Model] Add new model` — should be `[Model] Add <ModelName> ...`).
|
||||
|
||||
### Step 3: Categorize the PR
|
||||
|
||||
| Diff contains | PR type |
|
||||
|---------------|---------|
|
||||
| New files under `vllm_omni/model_executor/models/<name>/` | **New Model** |
|
||||
| Changes to `vllm_omni/diffusion/` | **Diffusion Model** |
|
||||
| `[Bugfix]` prefix or single-file fix | **Bug Fix** |
|
||||
| Perf/benchmark/throughput claims in commit msg or diff | **Performance** |
|
||||
| Everything else | **General** |
|
||||
|
||||
If multiple rows apply (e.g., a diffusion model is also a new model), union the checklists.
|
||||
|
||||
### Step 4: Run Checklist
|
||||
|
||||
Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type from [references/checklists.md](references/checklists.md). Each item produces ✓, ✗, or ⚠.
|
||||
|
||||
**Also run the Code-Quality sweep on every PR**, regardless of type or mode: the five diff-scoped checks in [references/code-quality.md](references/code-quality.md) — kwargs fragility, broad-except swallow, `Any`/wrong type hints, hot-path `.clone()`/`deepcopy`, and event-loop blocking — plus the advisory conventions (log level, structured logging, synchronization, cleanup, dependencies, naming) in [checklists.md](references/checklists.md). These count only lines the PR *adds* — the pre-existing backlog across the repo is out of scope.
|
||||
|
||||
**Also run the Examples-Policy check on every PR** using [references/examples-policy.md](references/examples-policy.md). Inspect only Python paths introduced by the diff. Treat a new model-, checkpoint-, vendor-, or family-specific Python example as blocking; do not report pre-existing example debt.
|
||||
|
||||
**Also run the Simplification pass on every PR** using [find-simplifications](../find-simplifications/SKILL.md). Keep it diff-scoped: inspect added or modified public APIs, state, abstractions, fallback and compatibility paths, data movement, and only the adjacent ownership needed to prove a candidate. Do not expand the pass into a repository-wide audit. Zero candidates is a valid pass; report only evidence-backed opportunities. Mark a candidate as blocking only when it proves a correctness or merge-readiness problem, such as newly unreachable code. Otherwise report it as a warning.
|
||||
|
||||
**Also confirm local pre-commit gates** from [docs/contributing/README.md](../../../docs/contributing/README.md#linting). GitHub Actions `SKIP` does not prove they passed. Full new-hook list: Omni SPDX (`vLLM-Omni project`); forbidden imports (stdlib `re`/`base64`, pickle, Hugging Face Hub API, Triton/TileLang); no new `torch.cuda` call sites; shellcheck (macOS/Windows must install the binary); mypy-3.10; test files have CI level + hardware marks; TTS `_tts_model_type` branches in `serving_speech.py` must not increase; markdownlint on `docs/`; Buildkite YAML schema. ✗ for growing `allowed_files` / `ALLOWED_FILES` or raising `MAX_MODEL_TYPE_BRANCHES` without review.
|
||||
|
||||
### Step 5: Print Report
|
||||
|
||||
```
|
||||
Pre-check report for <branch>
|
||||
|
||||
Mode: quick | full
|
||||
Type: <new-model | diffusion-model | bug-fix | perf | general>
|
||||
|
||||
Dimension Result
|
||||
───────────────── ──────
|
||||
PR title format ✓
|
||||
Code quality ⚠ 1 broad except, 2 Any hints
|
||||
Examples policy ✗ new model-specific Python example
|
||||
Simplification ⚠ helper duplicates an existing owner
|
||||
PR desc integrity ✓
|
||||
Registry/config ✓
|
||||
Dead code ⚠ 2 warnings
|
||||
Accuracy ✓
|
||||
Benchmark ✗ missing software versions
|
||||
|
||||
Verdict: 1 blocking | 2 warnings | recommend fixing ✗ before PR
|
||||
```
|
||||
|
||||
**Severity:**
|
||||
|
||||
| Mark | Meaning |
|
||||
|------|---------|
|
||||
| ✗ | Blocking — fix before opening PR |
|
||||
| ⚠ | Warning — consider fixing |
|
||||
| ✓ | Pass |
|
||||
| — | Skipped (not applicable) |
|
||||
|
||||
### Stop Here
|
||||
|
||||
Do not post comments, open PRs, or modify files. The report is for the contributor's terminal only.
|
||||
@@ -1,174 +0,0 @@
|
||||
# Pre-Check Checklists
|
||||
|
||||
Each item is a concrete check to run against the diff. ✗ = fix before PR. ⚠ = consider fixing.
|
||||
|
||||
---
|
||||
|
||||
## All PRs: Title Format (check first)
|
||||
|
||||
PR title must follow the project convention documented in `docs/contributing/README.md`:
|
||||
|
||||
| Prefix | Applies to |
|
||||
|--------|-----------|
|
||||
| `[Bugfix]` | Bug fixes |
|
||||
| `[CI/Build]` | Build or CI improvements |
|
||||
| `[Doc]` | Documentation changes |
|
||||
| `[Model]` | New/improved models (include model name) |
|
||||
| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
|
||||
| `[Kernel]` | CUDA/kernel changes |
|
||||
| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
|
||||
| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
|
||||
| `[Misc]` | Other changes (use sparingly) |
|
||||
|
||||
**Examples:**
|
||||
```
|
||||
✓ [Model] Add MiniCPM-o 4.5 support
|
||||
✓ [Bugfix] Fix Ovis image text encoder dtype
|
||||
✓ [Core] Qwen3-Omni performance optimization
|
||||
✓ [Hardware][Ascend] Fix VoxCPM2 on Ascend
|
||||
✗ Fix Ovis image text encoder dtype ← missing prefix
|
||||
✗ [bugfix] Fix Ovis image text encoder ← wrong case
|
||||
✗ [WIP] Add model ← WIP in title
|
||||
```
|
||||
|
||||
- [ ] **Title has valid prefix** — ✗ if missing prefix, wrong case, or WIP/Draft in title. For `[Model]` PRs, the title MUST also include the model identifier (e.g., `[Model] Add <ModelName> ...`). ⚠ if missing the model name.
|
||||
|
||||
---
|
||||
|
||||
## All PRs: Code Quality (run on every PR, quick & full)
|
||||
|
||||
Diff-scoped sweep for five fragility patterns that are pervasive in the existing codebase — **only lines the PR adds count**, not the pre-existing backlog. Detection commands, codebase examples, and full severity rules live in [code-quality.md](code-quality.md).
|
||||
|
||||
- [ ] **No new `**kwargs` string-lookup plumbing:** the diff adds no `**kwargs` + `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`. ⚠ any new instance; ✗ if a new string key is duplicated across ≥2 files without a shared constant, or unknown keys are silently dropped on a fail-fast path.
|
||||
- [ ] **No new broad exception swallow:** the diff adds no `except Exception:` / bare `except:`. ⚠ any new instance; ✗ for `except: pass` / `except Exception: return None` / `continue` on a fail-fast path (init, config validation, weight loading, request handling).
|
||||
- [ ] **No new `Any` / wrong type hints:** the diff adds no `: Any` / `-> Any` / untyped production signature, and no new `SimpleNamespace` in tests faking an object that already has a real typed stub. ⚠ any new instance; ✗ for a *wrong* (actively misleading) annotation or a `SimpleNamespace` test fake of a typed object.
|
||||
- [ ] **No new hot-path copy:** the diff adds no `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path. ⚠ any new instance; ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without a comment, or `deepcopy` of a scheduler / sampling-config on the request path.
|
||||
- [ ] **No new event-loop blocking:** the diff adds no `time.sleep` / blocking HTTP (`requests`, `urllib`) / `await` inside a held lock in `async def` code. ⚠ any new instance; ✗ for blocking sleep or HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock held across an `await`. Blocking inside a dedicated worker thread is fine.
|
||||
|
||||
Roll the ⚠/✗ counts into the report as a single **Code quality** dimension row.
|
||||
|
||||
---
|
||||
|
||||
## All PRs: Examples Policy (run on every PR, quick & full)
|
||||
|
||||
Inspect Python files introduced under `examples/` using
|
||||
[examples-policy.md](examples-policy.md). This is a diff-scoped ratchet: existing
|
||||
model-specific examples may be edited or removed without triggering the check.
|
||||
|
||||
- [ ] **No new model-specific Python examples:** the diff adds, copies, or renames no Python file whose path or behavior is specific to one model, checkpoint, vendor, or model family. ✗ for any violation. Route inference through a shared task/protocol entrypoint, model request behavior through `vllm_omni/model_extras`, and commands through task docs or `recipes/`.
|
||||
- [ ] **New generic example code is justified:** any new task-neutral Python file under `examples/` is an actual user-facing entrypoint rather than reusable production logic, a benchmark, an app, a setup tool, or a reproducer. ⚠ when the placement is questionable.
|
||||
|
||||
Roll the result into the report as a separate **Examples policy** dimension.
|
||||
|
||||
### Conventions (eyeball — no grep)
|
||||
|
||||
Judgment calls that don't grep cleanly — apply them while reading the diff, not as a mechanical check:
|
||||
|
||||
- **Right log level** — `info` is for end-user-visible events; routine internal events are `debug`; hot paths are `trace` or removed. Logging is not free.
|
||||
- **Structured fields over f-strings** — pass dynamic values as structured fields (`logger.info("msg", extra={...})`), not interpolated strings, so they stay greppable and parseable.
|
||||
- **No reflexive synchronization** — don't add `threading.Lock` / `asyncio.Lock` / `deepcopy` / re-wrap shared state unless there is real concurrent mutation; the GIL covers most Python code. Owners decide their own synchronization — don't pre-lock in a constructor.
|
||||
- **Context managers for cleanup** — use `with` / `contextlib` for files, locks, CUDA streams, and profilers; not raw `open()` + manual `.close()` or ad-hoc `finally`.
|
||||
- **No new dependencies without justification** — prefer the stdlib and existing vllm-omni deps; a new top-level dep needs a reason (check `pyproject.toml` / `requirements*.txt`).
|
||||
- **Naming / interface discipline** — don't rename or change interfaces just for taste in this PR, but do fix names that actively mislead (e.g. `serve` implying a long-running server, `Instance` where multiple instance kinds coexist).
|
||||
|
||||
---
|
||||
|
||||
## Bug Fix PRs
|
||||
|
||||
### Quick
|
||||
|
||||
- [ ] **PR body has 4 sections:** what broke (error message), repro steps, root cause, fix description
|
||||
- [ ] **Fix is minimal:** diff touches only files directly related to the bug. No unrelated refactors or style changes.
|
||||
- [ ] **Repro is runnable:** the PR body includes a copy-paste command that reproduces the bug
|
||||
- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
|
||||
|
||||
### Full (adds)
|
||||
|
||||
- [ ] **Regression test exists:** `git diff --name-only` includes at least one `tests/` file
|
||||
- [ ] **Regression test reproduces the original error:** the test would fail on main, pass on this branch
|
||||
- [ ] **Fix matches root cause exactly:** no "fix the symptom + something else" — if the root cause is dtype, the fix is dtype, not dtype + reformatting
|
||||
- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added. See the broad-except pattern in [code-quality.md](code-quality.md#2-broad-exception-swallow) for the full severity rule (⚠ any new broad catch; ✗ for a swallow on a fail-fast path).
|
||||
- [ ] **Upstream pattern match:** the fix follows the same pattern as existing code for similar cases (grep for analogous `from_pretrained` calls, etc.)
|
||||
- [ ] **Environment documented:** torch/transformers/vllm versions listed if the bug is version-dependent
|
||||
|
||||
---
|
||||
|
||||
## Performance PRs
|
||||
|
||||
### Quick
|
||||
|
||||
- [ ] **Before/after numbers in PR body:** at minimum, one metric (RTF, RPS, latency, VRAM) with main vs PR values
|
||||
- [ ] **Hardware specified:** GPU model, count, VRAM. "L4" or "H20" is sufficient. "tested on GPU" is not.
|
||||
- [ ] **Concurrency scaling:** if the optimization claims better throughput, numbers at 2+ concurrency levels
|
||||
- [ ] **No unexplained regressions:** if any metric goes down at any concurrency level, explain why
|
||||
- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
|
||||
|
||||
### Full (adds)
|
||||
|
||||
- [ ] **Software versions:** torch, CUDA, vllm, vllm-omni versions in PR body
|
||||
- [ ] **Warmup stated:** "1 warmup + 3 measured" or equivalent
|
||||
- [ ] **Stddev or range:** not just a single best run. "average of 2 rounds" is minimum.
|
||||
- [ ] **VRAM measured:** peak GPU memory before and after
|
||||
- [ ] **Benchmark script checked in:** under `tests/` or `examples/`, runnable by reviewer
|
||||
- [ ] **Exact command line in PR:** not pseudocode, not "run the benchmark"
|
||||
- [ ] **Pytest summary included:** `36 passed, 3 skipped, 17 warnings in 2897.66s` — proves the whole suite passes
|
||||
- [ ] **Config changes documented:** if `max_num_seqs`, `gpu_memory_utilization`, or deploy YAML changed, old and new values stated with reasoning
|
||||
|
||||
---
|
||||
|
||||
## New Model PRs
|
||||
|
||||
### Quick
|
||||
|
||||
- [ ] **PR body file list matches `git diff --name-only`:** every file path in the description exists in the diff
|
||||
- [ ] **Registry entries match claimed count:** count `_OMNI_MODELS` or `_DIFFUSION_MODELS` entries added vs claimed in body
|
||||
- [ ] **Pipeline config `custom_process_input_func` paths resolve:** each referenced module.function exists
|
||||
- [ ] **`__init__.py` exports match registry:** each registered class appears in `__init__.py` and `__all__`
|
||||
- [ ] **Shell script modes match README:** `run_curl.sh` supports every mode the README documents
|
||||
- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
|
||||
|
||||
### Full (adds)
|
||||
|
||||
- [ ] **Dead code scan (5 patterns):**
|
||||
1. Dead `forward()` — defined but never called through the pipeline path
|
||||
2. Dead factory/builder functions — instantiated nowhere
|
||||
3. Dead wrappers — thin wrappers with no callers
|
||||
4. Dead branch guards — `if version > X` where X is always satisfied
|
||||
5. Unused parameters — `__init__` args never accessed
|
||||
- [ ] **Copy-paste detection:** no string constant defined in 3+ files, no cross-module validation duplication, no near-identical shape coercion fns
|
||||
- [ ] **Import hygiene:** no re-export-only imports, no module-level side effects without docs, no `import os` in function body when already at top. Function-body imports are acceptable when the import is only reachable from a closure or factory return value (i.e., `import` would otherwise execute at module load even though the user may never trigger that path). Either lift to top-of-file or annotate with `# noqa: PLC0415`.
|
||||
- [ ] **Accuracy:** at least one test compares output against reference; audio = valid WAV at correct sample rate; determinism verified if seed claimed. ⚠ if missing and not explicitly deferred to a follow-up PR or RFC section; — if explicitly deferred (the deferral target SHOULD be linked in the PR body).
|
||||
- [ ] **Performance:** hardware, software versions, warmup, and metrics (RTF, VRAM, RPS, latency) stated. ⚠ if missing and not explicitly deferred; — if explicitly deferred.
|
||||
- [ ] **Benchmark settings:** model config (TP, PP, max_model_len, enforce_eager, quant), runtime config (batch, input/output spec), environment versions
|
||||
- [ ] **Benchmark script checked in + exact command line + pytest summary**
|
||||
|
||||
---
|
||||
|
||||
## Diffusion Model PRs
|
||||
|
||||
Diffusion models live under `vllm_omni/diffusion/`. In addition to the New Model checklist above, verify:
|
||||
|
||||
- [ ] **Transformer adapter follows contract:** inherits from `nn.Module`, implements `load_weights()`, uses `vllm_omni.diffusion.attention.layer.Attention`
|
||||
- [ ] **Pipeline has both offline and online paths:** `forward()` for batch generation, streaming path for online serving
|
||||
- [ ] **Cache-DiT integration:** at least one acceleration (TeaCache, FBCache, etc.) is wired up. ✗ for new **top-level** pipelines. For **subclass** pipelines that inherit `__call__` from a parent where Cache-DiT is already absent, downgrade to ⚠ and recommend filing a follow-up issue against the parent pipeline family.
|
||||
- [ ] **Parallelism config:** TP/SP/USP/CFG-Parallel options exposed in pipeline config
|
||||
- [ ] **Docs table updated:** model added to `docs/models/supported_models.md` with correct metadata
|
||||
- [ ] **E2E test exists:** at least one test exercises the full generate path
|
||||
|
||||
---
|
||||
|
||||
## General PRs
|
||||
|
||||
### Quick
|
||||
|
||||
- [ ] **PR body matches diff:** description doesn't claim changes not in the diff
|
||||
- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
|
||||
- [ ] **CI gates passing:** DCO, pre-commit, build — check `gh pr view --json statusCheckRollup` if a PR already exists
|
||||
- [ ] **Local pre-commit gates:** full list in [Linting](../../../../docs/contributing/README.md#linting). Blocking when the diff introduces: stale `vLLM project` SPDX; stdlib `re`/`base64` or pickle in library code; new `torch.cuda` call sites; `tests/**/test_*.py` without level+hardware marks; new `_tts_model_type` branches in `serving_speech.py`; invalid `.buildkite` YAML. macOS/Windows need a native `shellcheck`. ✗ for growing `CHECK_IMPORTS[*].allowed_files`, `ALLOWED_FILES`, or `MAX_MODEL_TYPE_BRANCHES` without review.
|
||||
|
||||
### Full (adds)
|
||||
|
||||
- [ ] **No new dead code in changed files:** scan for unreachable paths, unused imports
|
||||
- [ ] **Test coverage:** if core code changed (`engine/`, `stages/`, `connectors/`), tests are added or existing tests cover the path
|
||||
- [ ] **Import hygiene:** no redundant imports added
|
||||
- [ ] **No unrelated changes:** diff scope matches PR description
|
||||
@@ -1,179 +0,0 @@
|
||||
# Code-Quality Patterns
|
||||
|
||||
Five fragility patterns to sweep on **every** PR, in both quick and full mode. These are pervasive in the existing codebase (hundreds of pre-existing sites) — so the checks are **diff-scoped**: only lines the PR *adds* count. Whole-repo greps light up every PR with the backlog and are useless.
|
||||
|
||||
All commands assume:
|
||||
```bash
|
||||
BASE="$(git merge-base HEAD origin/main)"
|
||||
```
|
||||
|
||||
**Severity:** ⚠ for any *new* instance introduced by the diff; ✗ for the acute sub-cases defined per pattern. Acute ✗ findings belong in the report's blocking count.
|
||||
|
||||
---
|
||||
|
||||
## 1. `**kwargs` string-lookup passthrough
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
def forward(self, **kwargs):
|
||||
# BLOCKER-prone: raw dict + string keys duplicated across files
|
||||
if "runtime_additional_information" in kwargs and \
|
||||
"model_intermediate_buffer" not in kwargs:
|
||||
...
|
||||
```
|
||||
Real instances of this exact guard are copy-pasted across `cosyvoice3.py`, `indextts2_talker.py` (×2), `indextts2_s2mel_decoder.py`, `qwen3_tts_talker.py`, `qwen3_omni.py`. A typo in any string key silently no-ops; there is no type check, no IDE completion, and unknown keys are silently dropped.
|
||||
|
||||
### Why dangerous
|
||||
Raw `dict` + `**kwargs` + lists of param-name strings is **fragile by construction**: the compiler can't see a missing or misspelled key, so failures are silent. Each duplicated copy is a new place for the strings to drift.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `**kwargs` signature whose body does `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`.
|
||||
- ✗ when the diff introduces a **new** string key that is duplicated across ≥2 files without a shared module constant, or when `**kwargs` plumbing **silently drops unknown keys** on a fail-fast path (init, config validation, weight loading).
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E '\*\*kwargs|kwargs(\.get|\[)|"[a-z_]+"\s+in\s+kwargs'
|
||||
```
|
||||
|
||||
### Fix
|
||||
Prefer explicit typed params. The repo already has the right primitive: `msgspec.Struct` with `forbid_unknown_fields=True` rejects unknown keys instead of silently dropping them — see `_StructBase` in `vllm_omni/data_entry_keys.py:105` and `MoriPullRequest` in `vllm_omni/distributed/omni_connectors/connectors/mori_transfer_engine_connector.py:81`. A `@dataclass` or `TypedDict` works too.
|
||||
|
||||
If `**kwargs` is unavoidable (vLLM base-class compat), centralize the key strings as module-level constants behind **one** typed accessor, instead of repeating the literal across files.
|
||||
|
||||
---
|
||||
|
||||
## 2. Broad exception swallow
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# BLOCKER-prone: catches everything, returns None on any failure
|
||||
try:
|
||||
evt = build_event(...)
|
||||
return evt
|
||||
except Exception:
|
||||
return None
|
||||
```
|
||||
Real clusters: `vllm_omni/metrics/stats.py:240,274,367,450` and `vllm_omni/metrics/{modality,utils}.py` — all `except Exception: return None`.
|
||||
|
||||
### Why dangerous
|
||||
A generic `except Exception:` is **too flexible**. If we catch an exception type we don't expect, it is almost always an unhandled edge case that needs to be *fixed* — not swallowed into `None`/`pass`/`continue`. Broad catches turn fixable bugs into silent wrong behavior and make debugging impossible.
|
||||
|
||||
The repo backs this with ruff `BLE001` (broad-exception catch); it is currently suppressed via `# noqa: BLE001` in `vllm_omni/patch.py:413`. Prefer fixing over suppressing.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `except Exception:` / `except BaseException:` / bare `except:`.
|
||||
- ✗ for `except:` / `except Exception: pass|return None|continue` on a **fail-fast path** (init, config validation, weight loading, request handling, connector setup). This extends the Bug-Fix "No silent failure risk" ✗ in [checklists.md](checklists.md).
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# Any new broad catch (⚠ baseline):
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E 'except\s*(Exception|BaseException)?\s*:'
|
||||
# Note: this matches only the immediate-colon swallow forms (`except :`,
|
||||
# `except Exception:`) — it deliberately excludes specific catches like
|
||||
# `except ValueError:` and the `except Exception as e:` variant. Eyeball
|
||||
# the latter; `as e` usually means it is logged, but still verify.
|
||||
```
|
||||
|
||||
### Fix
|
||||
Catch the **specific** types you actually expect (`ValueError`, `KeyError`, `AttributeError`, `OSError`, `torch.cuda.OutOfMemoryError`, `msgspec.ValidationError`, …). At a genuine top-level / best-effort boundary (metrics, signal handlers), at minimum **log** the exception; never swallow into a bare `pass`/`return None` on a path that should fail loudly.
|
||||
|
||||
---
|
||||
|
||||
## 3. `Any` / wrong type annotations
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# ⚠ Any param + Any return — no contract, no narrowing
|
||||
def _extract_mm_output(engine_outputs: Any) -> dict[str, Any]:
|
||||
...
|
||||
```
|
||||
|
||||
### Why dangerous (root cause)
|
||||
`Any` disables the type checker for that position, so typos and wrong shapes pass silently. The repo has 636 `: Any` params, 244 `-> Any` returns, and **1,173 `SimpleNamespace` in `tests/`**. The `Any` leak is largely test-driven: vibe-coded unit tests fake objects with `SimpleNamespace`, which forces the production code consuming them to be typed `Any`, and the `Any` then propagates outward into public signatures.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `: Any` / `-> Any` / fully-untyped signature in production code.
|
||||
- ✗ for a **wrong** annotation (actively misleading — e.g. `-> bool` that returns `Optional[bool]`), or a *new* `SimpleNamespace` in a test that mimics an object which already has a real typed stub / `@dataclass` / `TypedDict` / `Protocol`.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# Any leaking into new signatures:
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E ':\s*Any\b|->\s*Any\b'
|
||||
# SimpleNamespace newly added in tests (the Any-leak source):
|
||||
git diff ${BASE}...HEAD -- "tests/" | grep "^+[^+]" | grep "SimpleNamespace"
|
||||
```
|
||||
|
||||
### Fix
|
||||
Replace `Any` with the concrete type, a `Protocol`, or a `Union`. If a type is genuinely dynamic, prefer `object` + `isinstance` narrowing over `Any` — `object` keeps the checker engaged. In tests, replace `SimpleNamespace` with the real class, a small `@dataclass`/`TypedDict`, or a `Protocol` + fake, so the system under test stays fully typed.
|
||||
|
||||
---
|
||||
|
||||
## 4. Unnecessary copy — `.clone()` / `copy.deepcopy` in hot paths
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# Per-step in a diffusion/AR loop: clones the full latent every iteration
|
||||
for t in timesteps:
|
||||
trajectory_latents.append(x_t.clone()) # O(N) GPU alloc + memcpy each step
|
||||
```
|
||||
Real clusters: `vllm_omni/diffusion/models/bagel/bagel_transformer.py` calls `x_t.clone()` ~9× across a trajectory loop (lines 1820–2186); the cache hooks `vllm_omni/diffusion/cache/teacache/hook.py:155` and `magcache/hook.py:182` clone `hidden_states` per step. Separately, `copy.deepcopy(self.scheduler)` / `copy.deepcopy(req.sampling_params)` is copy-pasted across diffusion pipelines — `pipeline_qwen_image.py:786`, `pipeline_hunyuan_image3.py:1904`, `pipeline_ltx2*.py` (×3), `pipeline_dreamzero.py:1085`, `diffusion_model_runner.py:398` — a full object-graph deepcopy on the request path. (Contrast `dreamzero/causal_wan_model.py:203`, which carries a `# No .clone() needed.` comment — the aware pattern.)
|
||||
|
||||
### Why dangerous
|
||||
A `.clone()` of a GPU tensor or a `copy.deepcopy` of a config object inside a per-step / per-request path is a hidden allocator: it reads as one Python line but triggers a GPU memcpy or a deep object-graph walk every iteration. Across 50 diffusion steps or hundreds of AR steps it dominates latency and VRAM bandwidth. `deepcopy` additionally ignores any custom device / `__copy__` logic and can drag tensors through CPU.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path.
|
||||
- ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without an explanatory comment, or `copy.deepcopy` of a scheduler / sampling-config on the request hot path.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E '\.clone\(\)|\.copy_\(|copy\.deepcopy|deepcopy\('
|
||||
```
|
||||
|
||||
### Fix
|
||||
Tensors: prefer a view (`x[..., :k]`, `torch.narrow`, `as_strided`) or write into a pre-allocated buffer with `dst.copy_(src)` instead of `.clone()`; if the caller no longer needs the original, `move`/`del` it rather than copy. Schedulers / configs: build a fresh lightweight instance per request, or split out the mutable per-request bits so the shared immutable part isn't deepcopied. If a copy is genuinely required (e.g. the source is mutated in place later), keep it and add a one-line comment saying why.
|
||||
|
||||
---
|
||||
|
||||
## 5. Blocking the asyncio event loop / lock held across `await`
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# BLOCKER-prone: blocking sleep / blocking HTTP / lock held across an await,
|
||||
# all inside an async (event-loop) function
|
||||
async def handle(self, req):
|
||||
with self._lock:
|
||||
resp = await self._client.call(req) # lock held across await -> serializes all requests
|
||||
time.sleep(0.1) # stalls the single loop for every concurrent request
|
||||
```
|
||||
|
||||
### Why dangerous
|
||||
vLLM-Omni serves over a single asyncio event loop. Any synchronous blocking work on an `async def` path — `time.sleep`, blocking HTTP (`requests`/`urllib`), heavy CPU work, or a lock held across an `await` — stalls the loop for *every* concurrent request, not just the blocking one. A lock held across `await` silently serializes the whole pipeline; blocking I/O tanks throughput and inflates tail latency.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `time.sleep(`, blocking HTTP call (`requests.get/post`, `urllib.request`, `urlopen`), or `await` inside a `with Lock` / `async with` critical section, in `async def` code.
|
||||
- ✗ for `time.sleep` or blocking HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock acquired and held across an `await` on the request path.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# New blocking primitives introduced by the diff — then eyeball whether each
|
||||
# sits in an async function on the loop vs a dedicated worker thread:
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E 'time\.sleep|requests\.(get|post|put|delete)|urllib\.request|urlopen'
|
||||
# Lock acquired with an await potentially inside the critical section (manual eyeball):
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" | grep -E 'async with|\.acquire\(\)'
|
||||
```
|
||||
Note: blocking inside a **dedicated worker thread is correct** — the repo already does this (`diffusion/model_loader/hub_prefetch.py`, `distributed/omni_connectors/kv_transfer_manager.py`, the `mooncake`/`mori` connectors poll with `time.sleep` on their own threads). The check targets the *event-loop* path only.
|
||||
|
||||
### Fix
|
||||
Move blocking work off the loop with `asyncio.to_thread(...)` / `run_in_executor`, or use a native async client (`httpx.AsyncClient` / `aiohttp`). Never `await` while holding a lock — drop the lock before the `await` and re-acquire after, or restructure so the critical section contains no `await`.
|
||||
|
||||
---
|
||||
|
||||
## Running the sweep
|
||||
|
||||
Run all five detections against `${BASE}...HEAD`, then for each hit decide ⚠ vs ✗ using the severity rules above. Roll the results into the report as a single **Code quality** dimension row (count of ⚠ and ✗), alongside the type-specific checklist from [checklists.md](checklists.md).
|
||||
@@ -1,68 +0,0 @@
|
||||
# Examples Python Policy
|
||||
|
||||
Prevent new model-specific Python examples without forcing cleanup of the
|
||||
existing backlog. Run this check on every PR in quick and full modes.
|
||||
|
||||
## Scope
|
||||
|
||||
Set the merge base as in the main workflow, then list Python paths introduced
|
||||
by the PR, including additions, copies, and renames:
|
||||
|
||||
```bash
|
||||
BASE="$(git merge-base HEAD origin/main)"
|
||||
git diff --name-status --diff-filter=ACR --find-renames --find-copies \
|
||||
"${BASE}"...HEAD -- 'examples/**/*.py'
|
||||
```
|
||||
|
||||
Inspect destination paths for copies and renames. Existing model-specific files
|
||||
that are only modified or deleted are grandfathered and must not be reported.
|
||||
|
||||
## Blocking classification
|
||||
|
||||
Mark ✗ when a newly introduced Python file does any of the following:
|
||||
|
||||
- Names or scopes a path to one model, checkpoint, vendor, or model family.
|
||||
- Hard-codes a model identifier, model-specific prompt, request shape, output
|
||||
conversion, or launch configuration behind an otherwise generic filename.
|
||||
- Duplicates an existing task/protocol runner to demonstrate one model.
|
||||
- Adds a per-model helper below a generic task hub such as `text_to_speech/`.
|
||||
|
||||
Do not infer safety from a generic filename alone. For example, a file named
|
||||
`text_to_audio.py` is still model-specific if it only implements one model's
|
||||
contract.
|
||||
|
||||
## Acceptable additions
|
||||
|
||||
A new Python example may pass when it is a genuinely model-neutral,
|
||||
user-facing task or protocol entrypoint. It must accept model selection through
|
||||
configuration and keep model-specific behavior out of the script.
|
||||
|
||||
Use ⚠ when the file is model-neutral but reusable logic appears misplaced under
|
||||
`examples/`. Route code by purpose:
|
||||
|
||||
| Code purpose | Preferred location |
|
||||
|---|---|
|
||||
| Model prompt, defaults, request/output adaptation | `vllm_omni/model_extras/` or production model modules |
|
||||
| Runnable model command and validation evidence | Task documentation or `recipes/` |
|
||||
| UI or interactive application | `apps/` |
|
||||
| Benchmark or evaluator | `benchmarks/` |
|
||||
| Regression reproducer | `tests/` |
|
||||
| Download, conversion, or setup utility | `tools/` |
|
||||
|
||||
If a model cannot use an existing shared runner, report the missing generic
|
||||
capability. Do not solve the gap by adding a model-specific Python example.
|
||||
|
||||
## Report format
|
||||
|
||||
Add one row to the pre-check report:
|
||||
|
||||
```text
|
||||
Examples policy ✓ no new Python example paths
|
||||
Examples policy ⚠ generic helper belongs in tools/
|
||||
Examples policy ✗ examples/offline_inference/new_model/end2end.py is model-specific
|
||||
Examples policy ✗ examples/offline_inference/x_to_y_model_name.py is model-specific
|
||||
```
|
||||
|
||||
For a blocker, name the path, the model-specific behavior, and the appropriate
|
||||
shared runner or destination. Do not propose deleting unrelated existing
|
||||
examples as part of the PR.
|
||||
@@ -1,458 +0,0 @@
|
||||
---
|
||||
name: production-add-diffusion-model
|
||||
description: Productionize a vLLM-Omni diffusion model after its Day-0 vertical slice works. Use when work requires official API parity and input limits, feature-combination evidence, online FP8, distributed layerwise offload, per-request Cache-DiT, CUDA/ROCm/NPU/XPU recipes, faster or sparse attention, fused operators, USP or disaggregation analysis, continuous batching and abort handling, long-running RPS stability, or accuracy/performance/reliability CI. For initial architecture porting, registry wiring, and basic weight loading, use add-diffusion-model first.
|
||||
---
|
||||
|
||||
# Productionizing a Diffusion Model
|
||||
|
||||
## Scope
|
||||
|
||||
Use this skill to move a working model integration from Day-0 support to a
|
||||
measured, fail-closed production deployment. It is deliberately separate from
|
||||
[`add-diffusion-model`](../add-diffusion-model/SKILL.md); do not modify or copy
|
||||
that skill when productionizing a model. If registry, pipeline, or basic loader
|
||||
work is incomplete, read that skill first and finish the Day-0 vertical slice.
|
||||
|
||||
MiniMax-H3 [PR #5691](https://github.com/vllm-project/vllm-omni/pull/5691)
|
||||
is the merged Day-0 case study. Its open optimization
|
||||
[issue #5700](https://github.com/vllm-project/vllm-omni/issues/5700) is an
|
||||
actively maintained roadmap and evidence map, not a list of already supported
|
||||
features. It can lag merged implementation: read its current revision, cited
|
||||
PRs, target source, and maintained recipes, then leave every unverified row
|
||||
`not tested` until the target revision has scoped evidence.
|
||||
|
||||
Read these references before the corresponding gate:
|
||||
|
||||
- [API and hardware recipes](references/api-and-recipes.md)
|
||||
- [Production feature patterns](references/feature-patterns.md)
|
||||
- [Performance patterns and code examples](references/performance-patterns.md)
|
||||
- [Serving, soak, and CI validation](references/production-validation.md)
|
||||
|
||||
## Non-negotiable evidence contract
|
||||
|
||||
Create a matrix whose row key is at least:
|
||||
|
||||
```text
|
||||
task x API mode x execution mode/capacity x shape/schedule x attention backend x
|
||||
packed layout x cache policy x quantization x offload mode x topology x
|
||||
hardware x dtype x output representation/transport
|
||||
```
|
||||
|
||||
Use exactly these states:
|
||||
|
||||
| State | Meaning |
|
||||
|---|---|
|
||||
| `validated` | The exact row passed correctness and measured deployment tests, with reproducible artifacts. |
|
||||
| `limited` | A bounded subset passed; the limitation and rejection/fallback behavior are explicit. |
|
||||
| `unsupported` | A stable architectural/platform restriction is proven and fails early with an actionable error. |
|
||||
| `not tested` | No adequate evidence exists. This is the default. |
|
||||
|
||||
Never infer compatibility from import success, server startup, another task,
|
||||
another card, or another topology. Record the model/checkpoint revision,
|
||||
vLLM-Omni commit, commands, request assets and hashes, seed/schedule/shape,
|
||||
raw output artifacts, environment, and result for every `validated` row.
|
||||
|
||||
## Workflow
|
||||
|
||||
Follow the gates in order. Keep a dense BF16 single-device path as the
|
||||
correctness oracle until every advertised fast path has parity evidence.
|
||||
|
||||
### Gate 0: Freeze the official contract
|
||||
|
||||
Pin the official implementation and checkpoint revision. Build a matrix for:
|
||||
|
||||
- constructor/call arguments, defaults, scheduler and sigma schedule;
|
||||
- task-to-source modality/count/order rules;
|
||||
- prompt, MIME, bytes, pixels, frames, duration, FPS, steps, seed, guidance,
|
||||
output count, and output format limits;
|
||||
- offline API and every public sync/async serving endpoint;
|
||||
- checkpoint partitions and task selection.
|
||||
|
||||
Normalize offline, sync, and async requests through one validation contract.
|
||||
Reject an invalid task/source combination before downloads, upload persistence,
|
||||
temporary files, async job creation, or engine submission. Use 400 for invalid
|
||||
semantics and 413 for payload limits; clean partial resources on every failure.
|
||||
See [API and hardware recipes](references/api-and-recipes.md).
|
||||
|
||||
### Gate 1: Re-prove correctness under production shapes
|
||||
|
||||
1. Make loading strict: track expected, loaded, unexpected, and intentionally
|
||||
ignored tensors. A fused destination is complete only after every source
|
||||
shard arrives. Missing retained parameters abort startup.
|
||||
2. Fix reference revision, asset hashes, seed, schedule, and shape. Compare
|
||||
component outputs, intermediate denoise latents, and final artifacts.
|
||||
3. Test unequal packed samples. Verify `cu_seqlens`, padding exclusion,
|
||||
modality/CFG ownership, split/gather boundaries, and one output per request.
|
||||
4. Test partial process groups with non-member ranks and group size smaller
|
||||
than world size. Do not construct a private group per component or request.
|
||||
5. Run negative API cases before positive E2E tests to prove side-effect-free
|
||||
rejection and cleanup.
|
||||
|
||||
### Gate 2: Wire shared fast primitives
|
||||
|
||||
Prefer explicit, testable vLLM-Omni/vLLM operators with native fallbacks.
|
||||
`torch.compile` remains a separately validated regional optimization; it is
|
||||
not a substitute for correct operator selection or lifecycle-safe boundaries.
|
||||
|
||||
#### RMSNorm + RoPE baseline
|
||||
|
||||
Use the shared cross-platform operators, preserving the official Q/K norm and
|
||||
RoPE order:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm
|
||||
from vllm_omni.diffusion.layers.rope import (
|
||||
RotaryEmbedding,
|
||||
apply_rope_to_qk,
|
||||
)
|
||||
|
||||
self.norm_q = RMSNorm(head_dim, eps=eps)
|
||||
self.norm_k = RMSNorm(head_dim, eps=eps)
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
|
||||
query = self.norm_q(query) # [B, S, Hq, D] or packed [T, Hq, D]
|
||||
key = self.norm_k(key) # [B, S, Hkv, D] or packed [T, Hkv, D]
|
||||
query, key = apply_rope_to_qk(
|
||||
self.rope,
|
||||
query,
|
||||
key,
|
||||
(cos, sin),
|
||||
)
|
||||
```
|
||||
|
||||
Verify `is_neox_style`, `half_head_dim`, partial rotary dimensions, cos/sin
|
||||
layout, and packed row ownership against the official model. `False` means
|
||||
interleaved/GPT-J style; do not assume the default is NeoX. Shared RMSNorm and
|
||||
RoPE are two dispatched operators, not automatically one fused kernel.
|
||||
|
||||
For packed non-interleaved RoPE, use the shared fused boundary when its contract
|
||||
matches the model:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
|
||||
|
||||
query, key = fused_qk_norm_rope(
|
||||
query, # [T, Hq, D]
|
||||
key, # [T, Hkv, D]
|
||||
self.norm_q.weight, # [D]
|
||||
self.norm_k.weight, # [D]
|
||||
rope_table, # [T, rotary_dim] = [cos | sin]
|
||||
self.norm_q.variance_epsilon,
|
||||
)
|
||||
```
|
||||
|
||||
The current CUDA fast path specializes BF16 `head_dim=128`, `rotary_dim=96`;
|
||||
the public function keeps an eager fallback for unsupported inputs. Verify the
|
||||
packed row/frequency contract and trace the actual fast path. Do not reshape an
|
||||
incompatible official RoPE layout merely to enter this kernel.
|
||||
|
||||
MiniMax-H3-style partial NeoX RoPE rotates a prefix and passes the suffix
|
||||
through:
|
||||
|
||||
```python
|
||||
self.q_norm = RMSNorm(head_dim, eps=eps, dtype=torch.bfloat16)
|
||||
self.k_norm = RMSNorm(head_dim, eps=eps, dtype=torch.bfloat16)
|
||||
self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)
|
||||
|
||||
def apply_partial_rope(self, x, freqs, rot_dim):
|
||||
x_rot, x_pass = x[..., :rot_dim], x[..., rot_dim:]
|
||||
cos, sin = torch.cos(freqs).to(x.dtype), torch.sin(freqs).to(x.dtype)
|
||||
return torch.cat((self.rope(x_rot, cos, sin), x_pass), dim=-1)
|
||||
```
|
||||
|
||||
The partial-RoPE snippet assumes `import torch` and lives on the owning
|
||||
attention module; adapt names and validated frequency shapes rather than
|
||||
copying it at module scope.
|
||||
|
||||
Call the modules normally so `CustomOp` selects CUDA/HIP/NPU/XPU/native
|
||||
implementations. Do not call `forward_cuda()` directly. Compare the shared
|
||||
path with native BF16 at operator, block, denoise-trajectory, and artifact
|
||||
levels before claiming either accuracy or speed.
|
||||
|
||||
#### Faster BF16 attention and fused projections
|
||||
|
||||
Use local TP head counts and role-aware shared attention:
|
||||
|
||||
```python
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=hidden_size,
|
||||
head_size=head_dim,
|
||||
total_num_heads=num_heads,
|
||||
total_num_kv_heads=num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.attn = Attention(
|
||||
num_heads=self.qkv_proj.num_heads,
|
||||
num_kv_heads=self.qkv_proj.num_kv_heads,
|
||||
head_size=head_dim,
|
||||
softmax_scale=head_dim**-0.5,
|
||||
causal=False,
|
||||
qkv_layout="BSND",
|
||||
role="self",
|
||||
prefix=prefix,
|
||||
)
|
||||
```
|
||||
|
||||
Prove the selected backend from runtime logs/trace and compare it with dense
|
||||
SDPA. Packed metadata must express real sample boundaries; do not build masks
|
||||
that the selected backend ignores. A backend that is faster for one shape or
|
||||
platform is not a global default. See [performance patterns](references/performance-patterns.md)
|
||||
for QKV, SwiGLU, AdaLN, sparse attention, redundancy, and fusion examples.
|
||||
|
||||
### Gate 3: Add online FP8 without breaking the loader
|
||||
|
||||
Route the active config to each component, then pass a stable runtime prefix
|
||||
and config into every quantizable vLLM linear:
|
||||
|
||||
```python
|
||||
def resolve_component_quant_config(quant_config, component):
|
||||
return quant_config.resolve(component) if hasattr(quant_config, "resolve") else quant_config
|
||||
|
||||
transformer_quant_config = resolve_component_quant_config(
|
||||
od_config.quantization_config,
|
||||
"transformer",
|
||||
)
|
||||
self.transformer = MyDiT(
|
||||
od_config,
|
||||
quant_config=transformer_quant_config,
|
||||
prefix="transformer",
|
||||
)
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=hidden_size,
|
||||
head_size=head_dim,
|
||||
total_num_heads=num_heads,
|
||||
total_num_kv_heads=num_kv_heads,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
```
|
||||
|
||||
Use `build_quant_config()` for direct construction/testing:
|
||||
|
||||
```python
|
||||
from vllm_omni.quantization import build_quant_config
|
||||
|
||||
quant_config = build_quant_config({
|
||||
"transformer": {"method": "fp8"},
|
||||
"text_encoder": None,
|
||||
"vae": None,
|
||||
"default": None,
|
||||
})
|
||||
```
|
||||
|
||||
Preserve each parameter's vLLM `weight_loader`; perform checkpoint layout
|
||||
conversion before invoking it. Account for every QKV/gate-up source shard.
|
||||
Keep precision-sensitive norm/modulation/embedders in BF16 unless separately
|
||||
proven, using the current `safe_quant_config` pattern where applicable.
|
||||
|
||||
Validate resident DiT FP8 first: prove which modules are quantized, HBM saved,
|
||||
BF16-vs-FP8 trajectory/artifact quality, latency, and throughput. Text encoder,
|
||||
VAE, other hardware, TP/cache/DLO combinations are separate rows.
|
||||
Pre-quantized, pruned, rotated, or adapter-modified checkpoints are separate
|
||||
model/loading lanes; compare them with the released BF16 model without calling
|
||||
the result a pure quantization ablation. Read
|
||||
[`../quantization/SKILL.md`](../quantization/SKILL.md) if present and prefer the
|
||||
target revision's implementation/docs when sibling guidance differs.
|
||||
|
||||
### Gate 4: Make every component offloadable
|
||||
|
||||
Declare topology instead of relying on heuristic discovery:
|
||||
|
||||
```python
|
||||
from typing import ClassVar
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
from vllm_omni.diffusion.offloader import OffloadPlan
|
||||
|
||||
class MyPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
_dit_modules: ClassVar[list[str]] = ["transformer"]
|
||||
_encoder_modules: ClassVar[list[str]] = ["text_encoder"]
|
||||
_vae_modules: ClassVar[list[str]] = ["vae"]
|
||||
_resident_modules: ClassVar[list[str]] = []
|
||||
_offload_plan: ClassVar[OffloadPlan] = OffloadPlan(
|
||||
on_demand_component_paths=frozenset({"text_encoder", "vae"}),
|
||||
block_attrs={"transformer": ("blocks",)},
|
||||
offload_submodules={"token_refiner": "blocks"},
|
||||
resident_dit_paths=frozenset({"transformer"}),
|
||||
encoder_block_attrs={"text_encoder": ("encoder.layers",)},
|
||||
)
|
||||
```
|
||||
|
||||
Add a test that resolves every dotted component/block path and fails if it is
|
||||
missing or is not the expected module/container; discovery may otherwise warn
|
||||
and skip. Validate ordinary layerwise offload and distributed layerwise
|
||||
offload (DLO) independently.
|
||||
|
||||
Keep component lifecycle and host-weight storage separate. `OffloadPlan`
|
||||
declares what can be streamed; the diffusion loader owns any `HostWeightPlan`
|
||||
and hands the exact prevalidated plan to DLO. Do not make the offloader rescan
|
||||
checkpoint files or duplicate loader name/shape/dtype decisions. Direct mmap is
|
||||
a fail-closed optimization: current preflight requires TP1, no HSDP, no online
|
||||
quantization, complete bindings, and represented transforms. Otherwise the
|
||||
ordinary loader remains authoritative.
|
||||
|
||||
If the target revision supports Host Weight Runtime (HWR), treat it as a
|
||||
separate, exact-identity startup/cache path rather than generic offload or
|
||||
zero-copy execution. Qualify `preferred` population/fallback and `required`
|
||||
consume-only failure independently; verify immutable manifests, final-layout
|
||||
restore, lease cleanup, corruption/quarantine, and node-local sharing. Current
|
||||
model contracts may be BF16 no-AllGather-only, so reject AllGather, online
|
||||
quantization, HSDP, LoRA/adapted weights, and unrecognized load formats unless
|
||||
the exact target revision explicitly adds them.
|
||||
|
||||
On small-HBM cards, record load/materialization peak, resident peak, encode,
|
||||
each denoise wave, decode, transient per-rank HBM, host PSS, H2D, and collective
|
||||
time. Test DLO AllGather and `--dlo-no-use-allgather` as different deployments.
|
||||
Sweep the supported resident-block policies and publish the non-dominated
|
||||
latency-HBM frontier; a single minimum-memory or fastest point is not the DLO
|
||||
trade-off. Keep request-specific preparation replica-local. Only weight
|
||||
materialization may enter the declared DLO collective group; never use WORLD
|
||||
for per-request work in a DP deployment.
|
||||
Direct checkpoint mmap preflight is limited to TP1 without HSDP or online
|
||||
quantization; TP>1 falls back to the ordinary TP-aware loader and can still feed
|
||||
DLO AllGather or no-AllGather after scoped E2E. HSDP remains incompatible with
|
||||
DLO AllGather. A target revision may allow finalized per-tensor online FP8
|
||||
through the ordinary loader and AllGather; keep every other online quantizer
|
||||
fail-closed, and require model/card/topology E2E before promotion.
|
||||
Use deploy YAML for DP under `--omni`; vLLM DP CLI flags are rejected.
|
||||
|
||||
### Gate 5: Add request-scoped acceleration safely
|
||||
|
||||
Server-wide Cache-DiT support does not imply a per-request quality API. Only
|
||||
add request policy after defining calibrated quality tiers and a safe hook
|
||||
lifecycle using `SupportsRequestScopedCacheDiT`, `CacheDiTRequestSpec`, and
|
||||
`RequestScopedCacheDiTRuntime`. Validate alternating and concurrent quality
|
||||
tiers, real cache hits, refresh/reset, success, error, disconnect, abort, and
|
||||
the next uncached request. Do not batch different quality policies unless the
|
||||
batch compatibility key and hook ownership make that safe.
|
||||
|
||||
Then test Cache-DiT against FP8, DLO, TP/SP, compile, and batching one
|
||||
combination at a time. See [production feature patterns](references/feature-patterns.md)
|
||||
for the lifecycle skeleton and compatibility matrix.
|
||||
|
||||
### Gate 6: Optimize only from profiles
|
||||
|
||||
Freeze one canonical workload manifest before profiling. Keep lossless runtime
|
||||
A/Bs, accelerated paths with numerical/quality trade-offs, and production
|
||||
topology studies in separate result lanes. Reconcile the client boundary as
|
||||
queue + encode + denoise + video/audio decode + output transport/codec +
|
||||
residual; do not hide a material residual in another stage.
|
||||
|
||||
Profile stage time, GPU kernels, CPU/GPU synchronization, allocations, H2D,
|
||||
output payload/copies, and collectives before choosing work. Apply and A/B one
|
||||
change at a time:
|
||||
|
||||
1. Remove synchronization and redundant denoise-loop work: hoist static prompt
|
||||
conditioning/token refinement, RoPE tables, masks/metadata, bounded AdaLN
|
||||
schedule projections, and reference scans; remove per-layer `.item()` and
|
||||
unused masks/allocations.
|
||||
2. Select the fastest correct BF16 attention backend for each role and shape.
|
||||
3. Prefer explicit fused QKV, gate-up + `SiluAndMul`, RMSNorm/RoPE, AdaLN, and
|
||||
layout/residual operators with guarded native fallbacks. Preserve every
|
||||
materialized dtype/rounding boundary consumed by the unfused graph.
|
||||
4. Enable sparse attention only when platform, shape, topology, realized
|
||||
sparsity, quality, fallback, and end-to-end speedup are all proven.
|
||||
5. Profile USP all-to-all count/bytes, packing, contiguous copies, overlap, and
|
||||
scaling efficiency. Audit replicated cross-attention before using
|
||||
`skip_sequence_parallel`. Qualify regular and accelerated Ulysses transport
|
||||
independently; activation logs, JIT/readiness time, workspace growth,
|
||||
stream ownership, maximum-shape warmup, and numerical drift are gates.
|
||||
Preserve the checkpoint's Q:KV head ratio when padding GQA for Ulysses, and
|
||||
validate strided QKV staging rather than assuming packed contiguity.
|
||||
6. Evaluate text-encoder or VAE disaggregation only if stage share, transfer
|
||||
volume, reuse/concurrency, and independent scaling justify it. Tiling,
|
||||
offload, or patch parallelism is not disaggregation.
|
||||
|
||||
Do not combine independent optimizations into one performance claim. Preserve
|
||||
raw before/after runs with fixed workload and warmup policy.
|
||||
|
||||
### Gate 7: Make serving interruptible and load-stable
|
||||
|
||||
Treat request-mode batching and step-wise execution as separate capabilities:
|
||||
|
||||
- request batching requires `supports_request_batch = True` and
|
||||
`DiffusionRequestBatch -> list[DiffusionOutput]`, exactly one output per
|
||||
request;
|
||||
- step execution requires the complete `SupportsStepExecution` contract:
|
||||
`prepare_encode`, `denoise_step`, `step_scheduler`, and `post_decode`.
|
||||
|
||||
Keep all mutable scheduler, RNG, latent, mask, cache, and step state request
|
||||
scoped. First validate `--step-execution --max-num-seqs 1` and step-boundary
|
||||
abort; only then test heterogeneous multi-request waves. Step continuous
|
||||
batching is experimental and is not automatically compatible with Cache-DiT.
|
||||
Before recommending it, predeclare a useful case and success threshold, then
|
||||
A/B it against request mode under the same arrival process. Structural support
|
||||
or a generic bridge is not latency, throughput, or cancellation evidence for a
|
||||
model.
|
||||
|
||||
Test queued and in-flight abort, client disconnect, timeout, OOM, worker error,
|
||||
and restart. Require one terminal result, idempotent cleanup, no post-decode
|
||||
after abort, no cache/temp/VRAM/request-ID leak, and a successful next request.
|
||||
Inject late worker results/exceptions after cancellation and prove the result
|
||||
pump stays alive instead of raising `InvalidStateError`. For large batched or
|
||||
long-window video outputs, freeze dtype, range, layout, contiguity, ownership,
|
||||
payload size, serializer/codec limits, and offline-versus-HTTP semantics.
|
||||
Validate device-side output preparation, D2H/IPC or shared handles, remote
|
||||
encoding, event-loop responsiveness, and client materialization as separate
|
||||
stages, including abort and consumer failure cleanup.
|
||||
A model/VAE callback that publishes ordered chunks is only a producer contract;
|
||||
it is not transport, backpressure, cancellation, public streaming, or
|
||||
time-to-first-frame evidence. Distinguish complete-response faster-than-playback
|
||||
(`client E2E / output duration <= 1`) from streaming and report first-chunk,
|
||||
steady cadence, finalization, and complete-artifact latency separately.
|
||||
Run below-, near-, and above-saturation mixed-RPS soaks and report success/error
|
||||
rate, throughput, queue time, memory slope, temp growth, and worker health.
|
||||
Report p50/p95/p99 only from a declared arrival model with enough measured
|
||||
requests for the claimed percentile. See
|
||||
[serving and CI validation](references/production-validation.md).
|
||||
|
||||
### Gate 8: Publish recipes and four independent CI tracks
|
||||
|
||||
For every target vendor/card/topology, publish exact environment, serve/deploy
|
||||
command, one complete JSON or multipart curl per supported task, expected
|
||||
output checks, benchmark command, warmup/repeat/concurrency policy, raw metrics,
|
||||
per-rank HBM/host PSS, accuracy artifact, and limitations. Mark each
|
||||
CUDA/ROCm/NPU/XPU row `validated`, `unsupported`, or `not tested`; never inherit
|
||||
support from the platform abstraction or another card. Re-run DLO on each
|
||||
small-HBM recipe.
|
||||
|
||||
Maintain separate CI gates:
|
||||
|
||||
1. Function: strict load, API/offline positive and negative contracts.
|
||||
2. Accuracy: fixed reference revision/assets/seed/schedule and scoped tolerance.
|
||||
3. Performance: fixed best deployment plus memory-constrained/DLO row, raw JSON,
|
||||
explicit warmup, enough samples for each claimed statistic, and owned
|
||||
regression thresholds.
|
||||
4. Reliability: long mixed-RPS soak, abort/disconnect/fault cleanup, memory and
|
||||
temporary-resource trend.
|
||||
|
||||
## Production Definition of Done
|
||||
|
||||
A model is production-ready only when:
|
||||
|
||||
- every official advertised task passes API parity, offline/online parity,
|
||||
strict loading, fixed-reference quality, and negative validation;
|
||||
- the recommended deployment row passes correctness, isolation, abort/error
|
||||
cleanup, benchmark, and soak gates on named hardware;
|
||||
- online FP8 and DLO are each implemented and either validated in scoped rows
|
||||
or explicitly limited/unsupported with fail-fast behavior;
|
||||
- per-request cache quality tiers have measured speed/quality frontiers and
|
||||
request isolation, if advertised;
|
||||
- CUDA/ROCm/NPU/XPU recipes describe evidence rather than inferred support;
|
||||
- Function, Accuracy, Performance, and Reliability CI have owners, artifacts,
|
||||
and actionable failure output;
|
||||
- accelerated paths identify every precision/quality trade-off, and the raw
|
||||
offline plus online encoded output contracts are versioned and tested;
|
||||
- all unverified task/backend/cache/quant/offload/topology/hardware combinations
|
||||
remain `not tested` and are not presented as supported.
|
||||
|
||||
Day-0 support is still a valid milestone, but title, docs, and matrix must say
|
||||
which representative task passed and which production gates remain.
|
||||
@@ -1,4 +0,0 @@
|
||||
interface:
|
||||
display_name: "Production Add Diffusion Model"
|
||||
short_description: "Productionize vLLM-Omni diffusion models"
|
||||
default_prompt: "Use $production-add-diffusion-model to take this integrated diffusion model from Day-0 support to evidence-backed production readiness."
|
||||
@@ -1,315 +0,0 @@
|
||||
# API Contract and Hardware Recipes
|
||||
|
||||
## Contents
|
||||
|
||||
1. [Freeze the upstream contract](#freeze-the-upstream-contract)
|
||||
2. [Validate before side effects](#validate-before-side-effects)
|
||||
3. [Task request templates](#task-request-templates)
|
||||
4. [Serving benchmark template](#serving-benchmark-template)
|
||||
5. [Hardware recipe contract](#hardware-recipe-contract)
|
||||
6. [Best deployment selection](#best-deployment-selection)
|
||||
|
||||
## Freeze the upstream contract
|
||||
|
||||
Pin three revisions independently:
|
||||
|
||||
- official source implementation commit or package version;
|
||||
- checkpoint/model-card revision and partition;
|
||||
- vLLM-Omni target commit.
|
||||
|
||||
Build a table before editing code. One row is one official task/partition.
|
||||
|
||||
| Field | Required content |
|
||||
|---|---|
|
||||
| Task and partition | Official task name, checkpoint subdirectory, task selection rules |
|
||||
| Call contract | Constructor and call arguments, types, required/optional, official defaults |
|
||||
| Prompt/input | Prompt count/length, modality, MIME, order, count, decoded shape |
|
||||
| Size/time | Width/height alignment, pixels, frames, duration, FPS, start offset |
|
||||
| Denoise | Scheduler, sigma schedule, steps, CFG/guidance, seed behavior |
|
||||
| Output | Count, format, codec/sample rate, return type |
|
||||
| API | Offline mapping, sync endpoint, async endpoint/job lifecycle |
|
||||
| Rejection | Status code and stable error class/message for every invalid boundary |
|
||||
|
||||
Official API parity includes limits and defaults, not just names. Do not add a
|
||||
generic default that changes a distilled checkpoint's fixed schedule, or accept
|
||||
an input combination the official task rejects.
|
||||
|
||||
For every accepted boundary, add the neighboring rejected values. Examples:
|
||||
`max_images` and `max_images + 1`, smallest/largest legal duration, aligned and
|
||||
misaligned dimensions, omitted and illegal steps, one and too many outputs.
|
||||
|
||||
## Validate before side effects
|
||||
|
||||
Use one normalization/validation path for offline, sync, and async requests.
|
||||
The ordering must be:
|
||||
|
||||
```text
|
||||
parse -> normalize aliases -> validate task/source matrix -> validate declared
|
||||
limits -> stream with byte caps -> inspect/decode with frame/pixel/duration caps
|
||||
-> persist only if needed -> submit to engine
|
||||
```
|
||||
|
||||
Fail before download, persistence, job creation, or engine submit whenever the
|
||||
available metadata is sufficient. For streamed uploads, enforce:
|
||||
|
||||
- total reference count and per-modality count before reading bodies;
|
||||
- per-file bytes while streaming;
|
||||
- aggregate bytes across all files;
|
||||
- decoded frames, duration, pixels, and media type before engine submission.
|
||||
|
||||
On rejection, close streams and remove every partial temporary resource. Use
|
||||
HTTP 400 for invalid task/field combinations and 413 for payload limits. Test
|
||||
both sync and async paths; an invalid async request must not create a job ID.
|
||||
|
||||
Do not trust client MIME alone. Sniff/validate content and keep ordered repeated
|
||||
multipart fields ordered. Avoid logging prompts, raw references, signed URLs,
|
||||
or base64 payloads.
|
||||
|
||||
## Task request templates
|
||||
|
||||
These are endpoint-shape templates, not model recipes. Replace placeholders
|
||||
with the exact official defaults and validated assets for the target model.
|
||||
Every production recipe must include all advertised tasks and an expected
|
||||
output check.
|
||||
|
||||
### Text to image: JSON
|
||||
|
||||
```bash
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/images/generations \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "<model>",
|
||||
"prompt": "<validated prompt>",
|
||||
"size": "1024x1024",
|
||||
"num_inference_steps": 30,
|
||||
"seed": 42,
|
||||
"response_format": "b64_json"
|
||||
}' | jq -er '.data[0].b64_json' | base64 -d > output.png
|
||||
file output.png
|
||||
```
|
||||
|
||||
### Image edit: multipart
|
||||
|
||||
```bash
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/images/edits \
|
||||
-F 'model=<model>' \
|
||||
-F 'prompt=<validated edit prompt>' \
|
||||
-F 'image=@/path/to/reference.png;type=image/png' \
|
||||
-F 'size=1024x1024' \
|
||||
-F 'num_inference_steps=30' \
|
||||
-F 'seed=42' \
|
||||
-F 'output_format=png' \
|
||||
| jq -er '.data[0].b64_json' | base64 -d > edited.png
|
||||
file edited.png
|
||||
```
|
||||
|
||||
Use repeated `image` fields only if the model's official input matrix allows
|
||||
multiple ordered sources. Never describe a file upload as JSON-only.
|
||||
|
||||
### Public request fields versus internal sampling fields
|
||||
|
||||
Use the field exposed by the target revision's endpoint schema; do not rename
|
||||
it from the pipeline's internal attribute. For example, the current video JSON
|
||||
and multipart contracts expose `extra_params`, while serving translates that
|
||||
object into `DiffusionSamplingParams.extra_args`. Other request paths may
|
||||
expose canonical `extra_args` or nest it under `extra_body`. Inspect the exact
|
||||
Pydantic model/FastAPI form dependency and exercise the generated curl before
|
||||
publishing a recipe. A deprecation warning in a different endpoint is not
|
||||
evidence that an unrecognized multipart form field works here.
|
||||
|
||||
Current endpoint map (re-check it at the target commit):
|
||||
|
||||
| Endpoint | Public model-specific field |
|
||||
|---|---|
|
||||
| `/v1/videos`, `/v1/videos/sync` | Multipart `extra_params` JSON string |
|
||||
| `/v1/images/generations` | JSON `extra_params` object |
|
||||
| `/v1/chat/completions` diffusion | `extra_body.extra_args` |
|
||||
| `/v1/images/edits` | Declared typed form fields only |
|
||||
| `/v1/audio/generate` | Declared typed JSON fields only |
|
||||
|
||||
Prefer declared top-level fields. Use a generic model-specific envelope only
|
||||
when the endpoint actually exposes it.
|
||||
|
||||
### Text to video: multipart
|
||||
|
||||
```bash
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
|
||||
-H 'Accept: video/mp4' \
|
||||
-F 'model=<model>' \
|
||||
-F 'prompt=<validated prompt>' \
|
||||
-F 'size=1280x720' \
|
||||
-F 'num_frames=81' \
|
||||
-F 'fps=16' \
|
||||
-F 'num_inference_steps=30' \
|
||||
-F 'seed=42' \
|
||||
-F 'extra_params={"task":"<official-task>"}' \
|
||||
-o output.mp4
|
||||
ffprobe -v error -show_entries stream=codec_name,width,height,r_frame_rate \
|
||||
-of json output.mp4
|
||||
```
|
||||
|
||||
### Image/video to video: multipart
|
||||
|
||||
```bash
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
|
||||
-H 'Accept: video/mp4' \
|
||||
-F 'model=<model>' \
|
||||
-F 'prompt=<validated prompt>' \
|
||||
-F 'num_frames=81' \
|
||||
-F 'fps=16' \
|
||||
-F 'num_inference_steps=30' \
|
||||
-F 'seed=42' \
|
||||
-F 'extra_params={"task":"<official-task>","start_time_seconds":0}' \
|
||||
-F 'input_reference=@/path/to/reference.png;type=image/png' \
|
||||
-o i2v.mp4
|
||||
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
|
||||
-H 'Accept: video/mp4' \
|
||||
-F 'model=<model>' \
|
||||
-F 'prompt=<validated prompt>' \
|
||||
-F 'num_frames=81' \
|
||||
-F 'num_inference_steps=30' \
|
||||
-F 'seed=43' \
|
||||
-F 'extra_params={"task":"<official-task>","start_time_seconds":0}' \
|
||||
-F 'input_reference=@/path/to/reference.mp4;type=video/mp4' \
|
||||
-o v2v.mp4
|
||||
```
|
||||
|
||||
For multiple references, repeat the model's accepted multipart field in
|
||||
official order. Include complete downloadable or repository fixtures plus
|
||||
SHA256 hashes. A placeholder-only recipe is not runnable evidence.
|
||||
|
||||
### Text to audio: JSON
|
||||
|
||||
```bash
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/audio/generate \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "<model>",
|
||||
"input": "<validated audio prompt>",
|
||||
"audio_length": 10.0,
|
||||
"num_inference_steps": 100,
|
||||
"seed": 42,
|
||||
"response_format": "wav"
|
||||
}' -o output.wav
|
||||
ffprobe -v error -show_entries stream=codec_name,sample_rate,channels \
|
||||
-of json output.wav
|
||||
```
|
||||
|
||||
Use `/v1/audio/generate` for general diffusion audio. Do not substitute the
|
||||
speech API unless the model is actually served through the speech contract.
|
||||
|
||||
### Async video lifecycle
|
||||
|
||||
Alongside `/v1/videos/sync`, include `POST /v1/videos`, status polling,
|
||||
content retrieval, cancellation/abort if public, and cleanup assertions. The
|
||||
same request must normalize to the same engine sampling parameters in both
|
||||
modes.
|
||||
|
||||
## Serving benchmark template
|
||||
|
||||
Run a smoke first, then fixed-concurrency and arrival-rate tests. Video task
|
||||
names for the harness are `t2v`, `i2v`, `ti2v`, or `v2v`; model-specific task
|
||||
selection belongs in `--extra-body` when required.
|
||||
|
||||
```bash
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--base-url http://127.0.0.1:8091 \
|
||||
--endpoint /v1/videos \
|
||||
--model '<model>' \
|
||||
--dataset random \
|
||||
--task t2v \
|
||||
--num-prompts 100 \
|
||||
--width 1280 --height 720 \
|
||||
--num-frames 81 --fps 16 \
|
||||
--num-inference-steps 30 \
|
||||
--request-rate 0.2 \
|
||||
--max-concurrency 4 \
|
||||
--warmup-requests 4 \
|
||||
--warmup-concurrency 4 \
|
||||
--extra-body '{"extra_params":"{\"task\":\"<official-task>\"}"}' \
|
||||
--output-file result.json
|
||||
```
|
||||
|
||||
The video backend currently stringifies each multipart value. Pre-serialize
|
||||
the nested `extra_params` object as above; passing it as a nested object would
|
||||
produce Python dict syntax rather than valid JSON. If the target revision
|
||||
changes the backend to JSON-encode dict/list values, update and smoke-test this
|
||||
command against that revision.
|
||||
|
||||
For i2v/v2v, use the harness's validated dataset/trace path or provide the
|
||||
documented media source; do not silently benchmark t2v and label it i2v. Keep
|
||||
prompt, assets, seed, dimensions, frames, steps, guidance, warmup, concurrency,
|
||||
and arrival process identical across A/B runs.
|
||||
|
||||
Keep three benchmark lanes distinct:
|
||||
|
||||
1. lossless runtime A/B with the released schedule/precision/attention policy;
|
||||
2. accelerated-path A/B that changes one adapter, quantization, cache, or
|
||||
sparsity policy and adds same-seed quality evidence;
|
||||
3. production-topology studies that vary placement or parallelism and report a
|
||||
latency/throughput/memory frontier rather than a kernel speedup.
|
||||
|
||||
For a fixed single-request A/B, retain every measured run and report median or
|
||||
mean with range after an explicit warmup; do not infer p95/p99 from a few
|
||||
repetitions. Tail latency requires a declared arrival process, request count,
|
||||
concurrency, and enough successful samples. Reconcile client E2E with queue,
|
||||
encoder, denoise, video/audio decode, D2H/IPC, codec, and residual timings.
|
||||
|
||||
For generated media, define `real-time` rather than relying on the label. A
|
||||
complete-response claim uses `client E2E / validated output duration <= 1` and
|
||||
does not imply streaming or low time-to-first-frame. Report first chunk/fragment
|
||||
and cadence separately when a public streaming route exists. Never add gains
|
||||
across benchmark lanes with different schedules, prompts, seeds, artifacts, or
|
||||
topologies.
|
||||
|
||||
The diffusion serving harness currently covers image/video task families, not
|
||||
`/v1/audio/generate`. For text-to-audio, use the target model's task-specific
|
||||
audio benchmark if one exists; otherwise add a serving harness that records
|
||||
request rate, concurrency, latency, generated-audio seconds per second, error
|
||||
rate, and output validation. Do not relabel the TTS harness or a video workload
|
||||
as diffusion-audio evidence.
|
||||
|
||||
## Hardware recipe contract
|
||||
|
||||
Create a separate row and runnable section for each vendor/card/topology:
|
||||
|
||||
| Evidence | Required detail |
|
||||
|---|---|
|
||||
| Identity | Vendor, exact SKU, device count, HBM, interconnect |
|
||||
| Software | Driver, CUDA/ROCm/CANN/oneAPI, framework/container, dependency versions |
|
||||
| Source | vLLM-Omni commit, official/checkpoint revision and hashes |
|
||||
| Deployment | dtype, attention backend per role, TP/SP/CFG/HSDP/DP, cache, quant, offload |
|
||||
| Commands | Complete deploy YAML/serve command, every task curl, benchmark command |
|
||||
| Method | Warmup exclusion, repetitions or request count, concurrency, arrival distribution, duration, statistic definitions |
|
||||
| Results | Success rate, p50/p95/p99, RPS/throughput per device, stage latency |
|
||||
| Memory | Load/materialize, resident, encode, denoise, decode, transient per-rank HBM and host PSS |
|
||||
| Quality | Metric/tolerance, artifact links/hashes, temporal/audio checks |
|
||||
| Output | Raw/offline dtype, range, layout and ownership; online codec, payload bytes, transport route and client boundary |
|
||||
| Limits | `validated`/`limited`/`unsupported`/`not tested`, fallback/rejection behavior |
|
||||
|
||||
Use current paths such as `recipes/<vendor>/`, `recipes/README.md`, supported
|
||||
models, and feature compatibility tables. Top-level documentation must link to
|
||||
the scoped matrix; it must not use an unqualified checkmark that hides a ROCm,
|
||||
NPU, XPU, task, topology, or dtype limitation.
|
||||
|
||||
DLO evidence is per card, especially on small-HBM devices. A DLO result on one
|
||||
CUDA card does not establish another CUDA SKU, ROCm, NPU, or XPU support.
|
||||
|
||||
## Best deployment selection
|
||||
|
||||
Benchmark a correctness-approved candidate matrix, not every theoretical
|
||||
combination. Choose the recommended row by the production objective—latency,
|
||||
throughput, cost, or small-HBM fit—and document that objective.
|
||||
|
||||
Do not recommend a row until it passes:
|
||||
|
||||
- all advertised task/API contracts;
|
||||
- fixed-reference quality;
|
||||
- request isolation and abort/error cleanup;
|
||||
- sustained arrival-rate tests;
|
||||
- per-rank HBM and host PSS limits;
|
||||
- hardware-specific accuracy and performance CI.
|
||||
|
||||
Keep the most memory-constrained DLO row even when it is not the fastest; it is
|
||||
a separate production recipe, not a failed latency candidate.
|
||||
@@ -1,462 +0,0 @@
|
||||
# Production Feature Patterns
|
||||
|
||||
## Contents
|
||||
|
||||
1. [Compatibility matrix](#compatibility-matrix)
|
||||
2. [Online FP8](#online-fp8)
|
||||
3. [Strict fused-weight loading](#strict-fused-weight-loading)
|
||||
4. [Distributed layerwise offload](#distributed-layerwise-offload)
|
||||
5. [Per-request Cache-DiT](#per-request-cache-dit)
|
||||
6. [Combination gates](#combination-gates)
|
||||
|
||||
## Compatibility matrix
|
||||
|
||||
Create rows at the granularity below. Add evidence links and a reason for every
|
||||
non-validated state.
|
||||
|
||||
| Task/shape | Execution mode/capacity | Backend/layout | Quant | Cache policy | Offload | Topology | Hardware/dtype | Output contract/transport | State | Evidence |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| `<task>/<shape>/<schedule>` | serial request / 1 | SDPA, padded | BF16 | none | resident | 1 device | `<card>` BF16 | raw float / local | `not tested` | — |
|
||||
| `<task>/<shape>/<schedule>` | step / N | `<fast>`, packed | BF16 | none | resident | `<TP/SP>` | `<card>` BF16 | encoded / subprocess | `not tested` | — |
|
||||
| `<task>/<shape>/<schedule>` | step / N | `<fast>`, packed | FP8 | high | DLO no-AG | `<TP/SP>` | `<card>` | device uint8 / SHM | `not tested` | — |
|
||||
|
||||
Initialize every row as `not tested`; the table is a work queue, not evidence.
|
||||
Promote the dense BF16 row to the oracle only after its scoped parity passes.
|
||||
Add one axis at a time. If the runtime cannot
|
||||
enforce a known-incompatible combination, add construction/admission
|
||||
validation before writing a recipe.
|
||||
|
||||
## Online FP8
|
||||
|
||||
### Component routing
|
||||
|
||||
The public builder accepts one method or a longest-prefix component map:
|
||||
|
||||
```python
|
||||
from vllm_omni.quantization import build_quant_config
|
||||
|
||||
config = build_quant_config({
|
||||
"transformer": {
|
||||
"method": "fp8",
|
||||
"ignored_layers": ["transformer.final_layer"],
|
||||
},
|
||||
"text_encoder": None,
|
||||
"vae": None,
|
||||
"default": None,
|
||||
})
|
||||
|
||||
assert config.resolve("transformer.blocks.0.attn.qkv_proj") is not None
|
||||
assert config.resolve("text_encoder.layers.0.mlp") is None
|
||||
assert config.resolve("vae.decoder.conv_out") is None
|
||||
```
|
||||
|
||||
Use actual runtime prefixes. Multi-DiT or nested pipelines may not use the
|
||||
generic names `transformer` and `vae`. A prefix that falls through to a default
|
||||
is not component validation. Add tests for every included and ignored prefix.
|
||||
|
||||
For CLI serving, first validate resident DiT-only FP8:
|
||||
|
||||
```bash
|
||||
vllm serve '<model>' --omni --quantization fp8 --max-num-seqs 1
|
||||
```
|
||||
|
||||
For scoped routing:
|
||||
|
||||
```bash
|
||||
vllm serve '<model>' --omni \
|
||||
--diffusion-quantization-config \
|
||||
'{"<dit-runtime-prefix>":{"method":"fp8","ignored_layers":["<exact-linear-prefix>"]},"<vae-runtime-prefix>":null,"default":null}' \
|
||||
--max-num-seqs 1
|
||||
```
|
||||
|
||||
Do not publish these placeholders. A recipe must contain the target model's
|
||||
verified prefixes.
|
||||
|
||||
### Quantizable module wiring
|
||||
|
||||
Pass `quant_config` and a stable prefix through every vLLM linear. Keep
|
||||
precision-sensitive modulation/norm layers safe unless proven otherwise.
|
||||
|
||||
```python
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=hidden_size,
|
||||
head_size=head_dim,
|
||||
total_num_heads=num_heads,
|
||||
total_num_kv_heads=num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn.qkv_proj",
|
||||
)
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size, intermediate_size],
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp.gate_up_proj",
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp.down_proj",
|
||||
)
|
||||
```
|
||||
|
||||
Set required checkpoint parameters to fail rather than silently initialize if
|
||||
the current loader supports `missing_param_init = "error"`.
|
||||
|
||||
### FP8 evidence
|
||||
|
||||
For each task/hardware row, capture:
|
||||
|
||||
- startup proves the FP8 quant method is attached to intended linear modules;
|
||||
- no unexpected BF16 fallback and no meta/uninitialized parameters;
|
||||
- same-seed component, trajectory, and final-artifact comparison with BF16;
|
||||
- HBM at load, resident, encode, denoise, decode, and peak;
|
||||
- cold load time; fixed-work warm-latency raw runs; and serving p50/p95/p99 plus
|
||||
throughput only from a declared arrival load with enough samples;
|
||||
- named ignored layers and why their BF16 retention is required.
|
||||
|
||||
DiT FP8 evidence does not cover the text encoder or VAE. CUDA evidence does not
|
||||
cover ROCm/NPU/XPU. A pre-quantized checkpoint is a different loading path from
|
||||
online FP8 and needs its own state. If it is also pruned, rotated, distilled, or
|
||||
adapter-modified, report that model delta explicitly; BF16-versus-artifact
|
||||
quality and speed are deployment comparisons, not pure quantization ablations.
|
||||
|
||||
## Strict fused-weight loading
|
||||
|
||||
Never use `param.data.copy_()` for quantizable vLLM parameters. Transform the
|
||||
checkpoint layout before invoking the parameter loader so the online quant
|
||||
loader remains outermost:
|
||||
|
||||
```python
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
param = params[target_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
|
||||
if target_name.endswith(".qkv_proj.weight"):
|
||||
packed_qkv = reorder_checkpoint_qkv(checkpoint_weight)
|
||||
weight_loader(param, packed_qkv)
|
||||
elif target_name.endswith(".gate_up_proj.weight"):
|
||||
gate, up = checkpoint_weight.chunk(2, dim=0)
|
||||
weight_loader(param, gate, 0)
|
||||
weight_loader(param, up, 1)
|
||||
else:
|
||||
weight_loader(param, checkpoint_weight)
|
||||
```
|
||||
|
||||
Track source shards per fused destination:
|
||||
|
||||
```python
|
||||
expected = {
|
||||
"blocks.0.attn.qkv_proj.weight": {"q", "k", "v"},
|
||||
"blocks.0.mlp.gate_up_proj.weight": {0, 1},
|
||||
}
|
||||
seen = {name: set() for name in expected}
|
||||
|
||||
# After mapping/loading each source:
|
||||
seen[target_name].add(shard_id)
|
||||
|
||||
incomplete = {
|
||||
name: sorted(shards - seen[name], key=str)
|
||||
for name, shards in expected.items()
|
||||
if shards - seen[name]
|
||||
}
|
||||
if incomplete:
|
||||
raise RuntimeError(f"incomplete fused checkpoint parameters: {incomplete}")
|
||||
```
|
||||
|
||||
Also compare retained model parameters with the returned loaded set. Explicit
|
||||
checkpoint ignores need a reason and test; an unknown or missing weight is not
|
||||
an ignore policy.
|
||||
|
||||
Load-time adapter fusion creates a dedicated model, not a request-switchable
|
||||
LoRA. Pin the base and adapter revisions, validate every low-rank and full-rank
|
||||
delta target, freeze the adapter-declared schedule/task, and fuse before
|
||||
sharding only when the loader contract proves identical rank-local weights.
|
||||
Reject offload or host-weight paths that bypass that fusion rather than serving
|
||||
the unfused base model silently.
|
||||
|
||||
## Distributed layerwise offload
|
||||
|
||||
### Declare component topology
|
||||
|
||||
Use `SupportsComponentDiscovery` and `OffloadPlan`. Validate all dotted paths
|
||||
at construction/test time because discovery may otherwise warn and skip.
|
||||
|
||||
Treat a future/requested `--layerwise-offload-components` selector as a public
|
||||
API only if it exists in the target revision. Today, express component topology
|
||||
with `OffloadPlan` (`on_demand_component_paths`, `block_attrs`,
|
||||
`encoder_block_attrs`, `resident_dit_paths`, and `offload_submodules`). Do not
|
||||
invent a CLI flag or claim component-selective DLO from discovery declarations
|
||||
alone; validate the selector separately when it lands.
|
||||
|
||||
```python
|
||||
from operator import attrgetter
|
||||
|
||||
import torch
|
||||
|
||||
def assert_offload_path(root, path):
|
||||
value = attrgetter(path)(root)
|
||||
if not isinstance(value, torch.nn.Module):
|
||||
raise TypeError(f"offload path {path!r} is not an nn.Module")
|
||||
return value
|
||||
|
||||
for path in (
|
||||
*pipeline._dit_modules,
|
||||
*pipeline._encoder_modules,
|
||||
*pipeline._vae_modules,
|
||||
*pipeline._offload_plan.on_demand_component_paths,
|
||||
):
|
||||
assert_offload_path(pipeline, path)
|
||||
```
|
||||
|
||||
For `block_attrs`, resolve the DiT first and then each declared block attribute.
|
||||
Require an indexable block container with at least one `nn.Module`. Validate
|
||||
`offload_submodules`, `resident_dit_paths`, and `encoder_block_attrs` similarly.
|
||||
|
||||
### Keep host storage loader-owned
|
||||
|
||||
Current DLO accepts a loader-produced `HostWeightPlan`. Pipeline authors should
|
||||
not instantiate this plan or add ad hoc mmap capability flags. The loader must
|
||||
preflight complete runtime/checkpoint bindings, names, shapes, dtypes, persistent
|
||||
buffers, and represented transforms before it skips ordinary materialization.
|
||||
The exact accepted plan is consumed once by DLO; a rejected plan records an
|
||||
observable fallback reason and continues through the ordinary loader.
|
||||
|
||||
Direct checkpoint mmap currently requires TP1 without HSDP or online
|
||||
quantization. TP>1 falls back to the ordinary TP-aware loader and may feed DLO
|
||||
AllGather or no-AllGather; the current design records a bounded DP2xTP2
|
||||
AllGather smoke, but that path does not receive direct-mmap host-page sharing.
|
||||
Per-tensor online FP8 must use the ordinary loader. Current revisions can admit
|
||||
its finalized weight/scale layout to DLO AllGather, including transposed runtime
|
||||
weights, while unsupported online quantizers still fail closed. Direct mmap
|
||||
remains unavailable, so capture the temporary full-model host materialization
|
||||
before rank sharding.
|
||||
|
||||
Host Weight Runtime is a distinct opt-in path. Current final-layout consumers
|
||||
use no-AllGather DLO and exact model-declared BF16 representations. In
|
||||
`preferred` mode, consume an exact local hit or canonically load and publish for
|
||||
a future startup; in `required` mode, consume only and fail on a miss or unusable
|
||||
artifact. HWR must not interact with DLO AllGather, online quantization, HSDP,
|
||||
LoRA/adapted weights, or an unrecognized load format unless the target revision
|
||||
adds an exact producer/restore contract. It is immutable node-local CPU backing,
|
||||
not zero-copy GPU execution; the DLO transport still owns staging or direct H2D.
|
||||
Qualify identity inputs, cold population, warm reuse, atomic concurrent
|
||||
publication, source digests, corruption/quarantine, lease cleanup, capacity,
|
||||
node-local PSS, H2D, and E2E latency. A host-memory saving that materially
|
||||
regresses transfer or request latency is a separate memory-first deployment.
|
||||
|
||||
### DLO deployment modes
|
||||
|
||||
Treat these paths separately:
|
||||
|
||||
| Path | Weight/runtime behavior | Required boundaries |
|
||||
|---|---|---|
|
||||
| DLO AllGather | Rank-sharded pinned host tensors; reconstruct layer through collective | TP>1 uses ordinary TP-aware loader output; HSDP and unsupported online quantizers are rejected; finalized per-tensor online FP8 is eligible only when recognized by the target revision; concurrent ranks must participate consistently |
|
||||
| DLO no-AllGather | Loader-approved checkpoint mmap or ordinary runtime tensors; each rank streams a complete block | Direct mmap is TP1/non-HSDP/non-online-quant only; TP/HSDP/online quant use ordinary runtime tensors and require scoped E2E |
|
||||
| SP + DLO | SP group can supply the DLO collective group | Packed boundaries and collective order require E2E |
|
||||
|
||||
Single-stage examples:
|
||||
|
||||
```bash
|
||||
# SP + DLO AllGather candidate; validate exact model/card before publishing.
|
||||
vllm serve '<model>' --omni \
|
||||
--enable-distributed-layerwise-offload \
|
||||
--usp 4 \
|
||||
--max-num-seqs 1
|
||||
|
||||
# TP + DLO rank-local candidate; not a support statement.
|
||||
vllm serve '<model>' --omni \
|
||||
--tensor-parallel-size 2 \
|
||||
--enable-distributed-layerwise-offload \
|
||||
--dlo-no-use-allgather \
|
||||
--max-num-seqs 1
|
||||
```
|
||||
|
||||
Under `--omni`, configure DP in a deploy YAML rather than passing vLLM DP CLI
|
||||
flags:
|
||||
|
||||
```yaml
|
||||
pipeline: <registered-pipeline>
|
||||
async_chunk: false
|
||||
data_parallel_size: 4
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0,1,2,3"
|
||||
max_num_seqs: 1
|
||||
enable_distributed_layerwise_offload: true
|
||||
dlo_use_allgather: true
|
||||
parallel_config:
|
||||
tensor_parallel_size: 1
|
||||
sequence_parallel_size: 1
|
||||
```
|
||||
|
||||
```bash
|
||||
vllm serve '<model>' --omni --deploy-config /path/to/dlo_dp4.yaml
|
||||
```
|
||||
|
||||
Keep the YAML next to the validated recipe. Match its schema to the target
|
||||
revision; do not assume a model accepts another pipeline's deploy config.
|
||||
|
||||
### DLO validation sequence
|
||||
|
||||
1. Resident BF16 reference.
|
||||
2. Ordinary layerwise offload parity.
|
||||
3. DLO enable, warmup, same-seed generation, disable/restart.
|
||||
4. AllGather and no-AllGather separately.
|
||||
5. Concurrent requests with different prompts and identical collective-affecting
|
||||
parameters; then reject or schedule incompatible steps/shapes safely.
|
||||
6. Queued/in-flight abort, error, idle rank, worker exit, and next request.
|
||||
7. TP, SP, cache, compile, online FP8 one axis at a time.
|
||||
8. Loader-plan selection/fallback reason, cold/warm storage lifecycle, and
|
||||
corruption/stale-entry recovery when mmap/cache backing is advertised.
|
||||
9. Replica-local request preparation plus DLO-group-only weight collectives in
|
||||
DP; exercise concurrent replicas to catch accidental WORLD rendezvous.
|
||||
10. Sweep resident block counts; publish raw latency/HBM pairs, dominated
|
||||
points, and the non-dominated Pareto frontier.
|
||||
11. Per-rank HBM, host PSS for the full process tree, H2D and collective trace.
|
||||
|
||||
On a small-HBM card, measure transient encode/VAE peaks as well as resident DiT
|
||||
weights. If a non-DiT component still exceeds HBM, combine only independently
|
||||
validated on-demand component staging, VAE tiling/patching, or another topology;
|
||||
do not hide the remaining peak behind the phrase “DLO enabled.”
|
||||
|
||||
## Per-request Cache-DiT
|
||||
|
||||
### Policy and lifecycle skeleton
|
||||
|
||||
First verify that the target revision contains the shared protocol/runtime.
|
||||
If not, follow the nearest current model's lifecycle without inventing a new
|
||||
public API.
|
||||
|
||||
The following is a **serial/exclusive request-mode sketch only**. Pipeline-wide
|
||||
hooks are mutable; concurrent requests need an admission/batching key and
|
||||
runner-owned transition serialization before this shape is safe.
|
||||
|
||||
```python
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.diffusion.cache.cachedit import (
|
||||
CacheDiTBackend,
|
||||
CacheDiTRequestSpec,
|
||||
RequestScopedCacheDiTRuntime,
|
||||
)
|
||||
|
||||
class MyPipeline(nn.Module):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
# ... components ...
|
||||
self._cache_dit_runtime = RequestScopedCacheDiTRuntime(self)
|
||||
|
||||
def adopt_cache_dit_backend(self, backend: CacheDiTBackend) -> None:
|
||||
self._cache_dit_runtime.adopt(
|
||||
backend,
|
||||
installation_key="my_model.generic",
|
||||
)
|
||||
|
||||
def is_cache_dit_enabled(self) -> bool:
|
||||
return self._cache_dit_runtime.is_enabled
|
||||
|
||||
def _cache_spec(self, quality, steps):
|
||||
if quality == "lossless":
|
||||
return None
|
||||
if quality == "high":
|
||||
return CacheDiTRequestSpec(
|
||||
installation_key="my_model.high.v1",
|
||||
cache_config=self._validated_high_config,
|
||||
num_inference_steps=steps,
|
||||
)
|
||||
raise ValueError(f"unsupported quality tier: {quality!r}")
|
||||
|
||||
def forward(self, req):
|
||||
quality, steps = validate_and_resolve_request(req)
|
||||
self._cache_dit_runtime.prepare(self._cache_spec(quality, steps))
|
||||
return self._run_validated_request(req)
|
||||
```
|
||||
|
||||
The snippet shows the model-owned transition for exclusive execution. Put
|
||||
disable/restore handling in the runtime/runner lifecycle that owns success,
|
||||
exception, disconnect, and asynchronous abort; a `try/finally` inside
|
||||
`forward()` alone cannot observe every server cancellation boundary.
|
||||
|
||||
The `installation_key` must include every policy dimension that changes hook
|
||||
installation; changes in step count can use runtime refresh. `None` disables
|
||||
installed hooks. Do not mutate the global startup cache config for a request.
|
||||
|
||||
Centralize cleanup in the owning request/runner boundary. An exception, abort,
|
||||
disconnect, or admission failure must not leave a changed hook policy visible
|
||||
to the next request. The exact owner depends on the target revision; add a
|
||||
failure-injection test rather than relying on `forward()` alone to observe
|
||||
asynchronous abort.
|
||||
|
||||
Example model-specific request form after calibration:
|
||||
|
||||
```bash
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
|
||||
-F 'model=<model>' \
|
||||
-F 'prompt=<prompt>' \
|
||||
-F 'quality=lossless' \
|
||||
-F 'num_inference_steps=<validated-steps>' \
|
||||
-o lossless.mp4
|
||||
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
|
||||
-F 'model=<model>' \
|
||||
-F 'prompt=<prompt>' \
|
||||
-F 'quality=high' \
|
||||
-F 'num_inference_steps=<validated-steps>' \
|
||||
-o high.mp4
|
||||
```
|
||||
|
||||
Do not expose these tier names merely because another model uses them.
|
||||
|
||||
### Cache-DiT evidence
|
||||
|
||||
For every tier/task/shape/schedule:
|
||||
|
||||
- show real cache hits/skipped blocks, not only “backend enabled” logs;
|
||||
- measure fixed-work speed and quality against lossless/native;
|
||||
- publish a speed/quality frontier and chosen tier mapping;
|
||||
- alternate lossless/high/lossless and vary steps to prove restore/refresh;
|
||||
- interleave tasks and shapes; run concurrent requests only when scheduling
|
||||
guarantees exclusive compatible hook state;
|
||||
- inject validation error, denoise error, disconnect, and abort, then prove the
|
||||
next request starts with its requested policy;
|
||||
- bound any cached tensors and clear request-local state.
|
||||
|
||||
## Combination gates
|
||||
|
||||
Start every combination as `not tested`.
|
||||
|
||||
| Combination | Default production posture |
|
||||
|---|---|
|
||||
| Direct checkpoint mmap + TP>1/HSDP/online quant | Preflight falls back to the ordinary loader; do not claim direct-mmap savings |
|
||||
| Per-tensor online FP8 + DLO AllGather | Generic compatibility exists for finalized weights/scales; each model/card/topology remains a candidate until E2E, and other online quantizers fail closed |
|
||||
| Online FP8 + DLO no-AllGather | Candidate only after model/card/topology E2E |
|
||||
| TP>1 + DLO AllGather | Ordinary TP-aware loader only; bounded DP2xTP2 smoke exists, but each new model/card/topology still needs E2E |
|
||||
| TP>1 + DLO no-AllGather | Ordinary TP-aware loader only; candidate with limited generic coverage |
|
||||
| HSDP + DLO AllGather | Reject; it would double-shard HSDP-managed parameters |
|
||||
| HSDP + DLO no-AllGather | Configuration-compatible candidate with limited E2E coverage |
|
||||
| Cache-DiT + online FP8 | Separate trajectory, hit-rate, HBM, latency test |
|
||||
| Cache-DiT + DLO | Verify hooks, streamed blocks, memory peaks, abort cleanup |
|
||||
| Cache-DiT + step execution | Do not advertise until lifecycle/batching support is explicit |
|
||||
| Sparse attention + Ring | Verify backend execution; reject silently ignored sparse settings |
|
||||
| Load-time fused adapter + offload/HWR | Reject unless the selected loader path applies and validates the same fusion before sharding/storage |
|
||||
| Packed attention + any topology | Unequal-sample boundary parity is mandatory |
|
||||
| Device-side uint8 preparation + offline caller | Version the dtype/range/layout change; HTTP MP4 parity does not preserve the raw tensor contract |
|
||||
| Device-side preparation + remote codec | Validate route selection, strided/contiguous layouts, payload bytes, ordering, fallback, and encoded-media parity |
|
||||
|
||||
A startup-only test never upgrades a combination. Require request output,
|
||||
parity/quality, actual fast-path evidence, resource cleanup, and a benchmark on
|
||||
the named hardware.
|
||||
@@ -1,481 +0,0 @@
|
||||
# Performance Patterns and Code Examples
|
||||
|
||||
## Contents
|
||||
|
||||
1. [Measurement loop](#measurement-loop)
|
||||
2. [Faster BF16 attention](#faster-bf16-attention)
|
||||
3. [Sparse attention](#sparse-attention)
|
||||
4. [Fusion-first operator patterns](#fusion-first-operator-patterns)
|
||||
5. [Remove redundant computation](#remove-redundant-computation)
|
||||
6. [USP communication optimization](#usp-communication-optimization)
|
||||
7. [Text encoder and VAE disaggregation](#text-encoder-and-vae-disaggregation)
|
||||
8. [Performance acceptance](#performance-acceptance)
|
||||
|
||||
## Measurement loop
|
||||
|
||||
Do not start with a list of optimizations. Start with a fixed correctness and
|
||||
performance workload:
|
||||
|
||||
```text
|
||||
model/checkpoint revision + prompt/media hashes + seed + dimensions/frames +
|
||||
scheduler/sigma/steps/guidance + task + dtype + backend + topology + hardware
|
||||
```
|
||||
|
||||
Capture:
|
||||
|
||||
- stage time: load, encode, conditioning, denoise, VAE decode/postprocess;
|
||||
- GPU trace: kernel duration/count, launch gaps, allocations, H2D, collectives;
|
||||
- CPU trace: preprocessing, per-layer sync, Python dispatch, temporary files;
|
||||
- memory: per-rank HBM peaks and process-tree host PSS;
|
||||
- output: device preparation, D2H/IPC, payload bytes, codec wall/process CPU,
|
||||
event-loop wait, client materialization, and signed residual;
|
||||
- serving: queue time, throughput, errors, request mix, and distribution samples.
|
||||
|
||||
For a fixed-work single-request A/B, run one explicit warmup policy and at
|
||||
least three measured repetitions, keep every raw run, and report median or mean
|
||||
with range. Do not derive p95/p99 from three samples. For tail latency, declare
|
||||
the arrival model and collect enough successful requests for the percentile and
|
||||
confidence claimed. Implement one optimization at a time, re-run parity, and
|
||||
report both local kernel and end-to-end deltas. A kernel win that moves no E2E
|
||||
metric is not a production speed claim.
|
||||
|
||||
Use the pipeline profiler already exposed by the target revision when
|
||||
available, and use PyTorch/CUDA/ROCm/NPU/XPU profiling tools appropriate to the
|
||||
platform. Do not force every platform through a CUDA-only workflow.
|
||||
|
||||
## Faster BF16 attention
|
||||
|
||||
### Shared attention wiring
|
||||
|
||||
Use the shared role-aware attention layer and give it the true local head
|
||||
counts and QKV layout:
|
||||
|
||||
```python
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
|
||||
class SelfAttention(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size,
|
||||
head_dim,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
quant_config=None,
|
||||
prefix="",
|
||||
):
|
||||
super().__init__()
|
||||
self.head_dim = head_dim
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=hidden_size,
|
||||
head_size=head_dim,
|
||||
total_num_heads=num_heads,
|
||||
total_num_kv_heads=num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.attention = Attention(
|
||||
num_heads=self.qkv_proj.num_heads,
|
||||
num_kv_heads=self.qkv_proj.num_kv_heads,
|
||||
head_size=head_dim,
|
||||
softmax_scale=head_dim**-0.5,
|
||||
causal=False,
|
||||
qkv_layout="BSND",
|
||||
role="self",
|
||||
prefix=prefix,
|
||||
)
|
||||
self.out_proj = RowParallelLinear(
|
||||
num_heads * head_dim,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
|
||||
def forward(self, x, attn_metadata=None):
|
||||
qkv, _ = self.qkv_proj(x)
|
||||
q_size = self.qkv_proj.num_heads * self.head_dim
|
||||
kv_size = self.qkv_proj.num_kv_heads * self.head_dim
|
||||
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
q = q.unflatten(-1, (self.qkv_proj.num_heads, self.head_dim))
|
||||
k = k.unflatten(-1, (self.qkv_proj.num_kv_heads, self.head_dim))
|
||||
v = v.unflatten(-1, (self.qkv_proj.num_kv_heads, self.head_dim))
|
||||
out = self.attention(q, k, v, attn_metadata)
|
||||
out = out.flatten(-2)
|
||||
out, _ = self.out_proj(out)
|
||||
return out
|
||||
```
|
||||
|
||||
Add Q/K norm and RoPE in the official order before the `Attention` call. Keep
|
||||
checkpoint mapping for the fused QKV and output projection complete.
|
||||
|
||||
### Backend qualification
|
||||
|
||||
For each role (`self`, cross-attention category, token refiner, etc.), record:
|
||||
|
||||
- actual selected backend from logs/trace;
|
||||
- supported dtype, head size, QKV layout, mask/metadata, packed varlen behavior;
|
||||
- platform and compute capability;
|
||||
- TP/SP/Ring/AllGather-KV restrictions;
|
||||
- fallback behavior and whether fallback changes the claimed performance.
|
||||
|
||||
Use unequal packed samples and compare with dense SDPA. Verify `cu_seqlens`,
|
||||
max sequence length, padding exclusion, modality boundaries, CFG ownership, and
|
||||
output slicing. Do not pass an `attn_mask` and assume it is consumed. Inspect
|
||||
the backend contract and trace.
|
||||
|
||||
TRTLLM, Flash Attention, cuDNN, NPU varlen, and other backends are independent
|
||||
qualification rows. The shared `Attention` class does not make every backend
|
||||
packed-safe or faster automatically. Ring and hybrid paths require their own
|
||||
boundary parity. AllGather-KV and specific backend combinations may fail fast;
|
||||
keep that behavior visible in docs.
|
||||
|
||||
## Sparse attention
|
||||
|
||||
Sparse attention is a backend/model/platform optimization, not a synonym for
|
||||
low-precision dense attention. Enable it only after dense BF16 parity.
|
||||
|
||||
For every candidate task/shape/topology:
|
||||
|
||||
1. State the sparsity policy, selected backend/kernel, block/head dimensions,
|
||||
supported hardware, external kernel/package revision, and fallback.
|
||||
2. Prove the sparse kernel executed and report realized sparsity; an enabled
|
||||
flag is insufficient.
|
||||
3. Compare same-seed per-block outputs, denoise trajectory, and final artifact
|
||||
with dense attention.
|
||||
4. Add modality-specific metrics: temporal consistency and audio/video sync for
|
||||
video, spectral/audio quality for audio, and prompt/image metrics as relevant.
|
||||
5. Report attention time and full E2E time. Include preprocessing/index-build
|
||||
overhead, peak memory, and the exact dense control.
|
||||
6. Validate each task and shape. Do not extrapolate a T2V result to I2V/Ref2V,
|
||||
or NPU sparse support to CUDA/ROCm/XPU.
|
||||
|
||||
If Ring bypasses the selected backend or a sparse setting would be silently
|
||||
ignored, reject the combination. Prefer a visible dense fallback only when the
|
||||
recipe labels the row `limited` and does not claim sparse speed.
|
||||
|
||||
## Fusion-first operator patterns
|
||||
|
||||
Prefer current shared/custom operators with explicit guards and native
|
||||
fallbacks. Prove parity at operator, block, trajectory, and artifact levels,
|
||||
then use a trace to prove fewer launches/bytes or lower latency.
|
||||
|
||||
### Q/K RMSNorm + RoPE
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding, apply_rope_to_qk
|
||||
|
||||
self.norm_q = RMSNorm(head_dim, eps=eps)
|
||||
self.norm_k = RMSNorm(head_dim, eps=eps)
|
||||
# Example only: set these booleans from the official checkpoint contract.
|
||||
use_neox_layout = False
|
||||
cos_sin_are_half_dim = True
|
||||
self.rope = RotaryEmbedding(
|
||||
is_neox_style=use_neox_layout,
|
||||
half_head_dim=cos_sin_are_half_dim,
|
||||
)
|
||||
|
||||
q = self.norm_q(q)
|
||||
k = self.norm_k(k)
|
||||
q, k = apply_rope_to_qk(self.rope, q, k, (cos, sin))
|
||||
```
|
||||
|
||||
Replace the illustrative booleans with the official layout in real code.
|
||||
Preserve partial rotary dimensions by splitting the rotated prefix and concatenating
|
||||
the pass-through suffix. If 3D cos/sin includes a batch axis, the current
|
||||
shared path assumes the values are shared across the batch; per-sample
|
||||
different positions require a proven representation/path.
|
||||
|
||||
For packed `[tokens, heads, head_dim]` tensors with a non-interleaved
|
||||
`[cos | sin]` table, the current shared fused boundary is:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
|
||||
|
||||
q, k = fused_qk_norm_rope(
|
||||
q,
|
||||
k,
|
||||
self.norm_q.weight,
|
||||
self.norm_k.weight,
|
||||
rope_table,
|
||||
self.norm_q.variance_epsilon,
|
||||
)
|
||||
```
|
||||
|
||||
Its CUDA fast path currently specializes BF16 `head_dim=128` and
|
||||
`rotary_dim=96`; other valid inputs take the eager implementation. The separate
|
||||
shared RMSNorm and RotaryEmbedding still provide the general cross-platform
|
||||
baseline. Treat NPU/MUSA and other vendor fusions as independent qualification
|
||||
rows, and do not generalize a model-specific Triton kernel without compatible
|
||||
shape, layout, frequency partition, dtype, and platform evidence.
|
||||
|
||||
### Fused QKV projection
|
||||
|
||||
Use `QKVParallelLinear` as shown above. Qualification includes:
|
||||
|
||||
- `num_heads` and `num_kv_heads` divisibility by TP;
|
||||
- local split sizes from the layer, never reusing global counts after TP;
|
||||
- Q/K/V checkpoint order conversion before the vLLM loader;
|
||||
- all Q, K, and V shards present;
|
||||
- quantization scale/loader mapping when FP8 or another method is enabled.
|
||||
|
||||
### Gate-up projection + fused SwiGLU
|
||||
|
||||
```python
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
input_size=hidden_size,
|
||||
output_sizes=[intermediate_size, intermediate_size],
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
self.down_proj = RowParallelLinear(
|
||||
input_size=intermediate_size,
|
||||
output_size=hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
```
|
||||
|
||||
Map both checkpoint `gate_proj` and `up_proj` shards into the merged parameter
|
||||
and require both. A merged projection followed by `F.silu(gate) * up` is not
|
||||
the same claim as the `SiluAndMul` fused activation kernel; identify what the
|
||||
trace actually uses.
|
||||
|
||||
### AdaLN
|
||||
|
||||
Use a shared layer only when its mathematical/return contract matches the
|
||||
official model:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
|
||||
|
||||
self.adaln = AdaLayerNorm(
|
||||
hidden_size,
|
||||
elementwise_affine=False,
|
||||
eps=1e-6,
|
||||
)
|
||||
x = self.adaln(x, scale[:, None, :], shift[:, None, :])
|
||||
```
|
||||
|
||||
`AdaLayerNormZero`, `AdaLayerNormZeroSingle`, and continuous variants include
|
||||
different projections, broadcasts, and tuple outputs; they are not drop-in
|
||||
replacements. A shared AdaLN may have an NPU fast path while using native ops on
|
||||
CUDA/HIP/XPU. Do not call it a universal fused AdaLN kernel.
|
||||
|
||||
For sigma schedules with a small finite set of timesteps, profile whether
|
||||
precomputing the timestep/AdaLN projection once per request is worthwhile.
|
||||
Cache keys must include the exact schedule/config/device/dtype and stay bounded.
|
||||
|
||||
### Layout, residual, and other candidates
|
||||
|
||||
After profiling, inspect:
|
||||
|
||||
- bias/dropout/add or gated residual chains;
|
||||
- norm + linear or norm + modulation boundaries;
|
||||
- repeated transpose/reshape/contiguous around attention;
|
||||
- VAE normalization/activation/convolution boundaries;
|
||||
- MoE routing and fused experts for models that actually use MoE.
|
||||
|
||||
For fused residual-plus-norm or modulation chains, preserve the unfused
|
||||
materialization boundary. If the residual was stored in BF16 before RMSNorm,
|
||||
round to BF16 and promote again inside the fused kernel; using the unrounded
|
||||
FP32 temporary changes the denoising trajectory even when the stored residual
|
||||
looks correct.
|
||||
|
||||
Keep DLO materialize/release hooks, process-group collectives, request cache
|
||||
transitions, and cleanup outside compiled/fused regions unless lifecycle safety
|
||||
is explicitly tested. Compile only stable regions and retain eager fallback.
|
||||
|
||||
## Remove redundant computation
|
||||
|
||||
### Hoist request invariants
|
||||
|
||||
Audit the denoise loop for work that depends only on the request rather than
|
||||
the current timestep:
|
||||
|
||||
```python
|
||||
# Before the denoise loop.
|
||||
condition = self.condition_proj(text_embeddings)
|
||||
refined_condition = self.token_refiner(condition, packed_metadata)
|
||||
rope_tables = self.build_rope_tables(position_ids)
|
||||
attn_metadata = self.build_attention_metadata(layout)
|
||||
|
||||
for step, sigma in enumerate(sigmas):
|
||||
timestep_state = self.prepare_timestep_state(sigma)
|
||||
noise = self.transformer(
|
||||
latents,
|
||||
refined_condition,
|
||||
rope_tables=rope_tables,
|
||||
attn_metadata=attn_metadata,
|
||||
timestep_state=timestep_state,
|
||||
)
|
||||
latents = self.scheduler_step(noise, sigma, latents)
|
||||
```
|
||||
|
||||
Candidates include prompt conditioning, token-refiner output, reference
|
||||
encoding/scans, position IDs/RoPE tables, packed masks/metadata, static CFG
|
||||
branches, and finite-schedule AdaLN projections. First prove they are invariant
|
||||
for all official tasks, LoRA/adapters, prompt-update features, shapes, and
|
||||
schedules. In continuous batching, store request-specific results on request
|
||||
state rather than the pipeline singleton.
|
||||
|
||||
### Eliminate per-layer synchronization
|
||||
|
||||
Avoid device-to-host scalar extraction such as `.item()` inside every layer:
|
||||
|
||||
```python
|
||||
# Bad: forces a GPU/CPU synchronization in every attention layer.
|
||||
max_len = int(cu_seqlens[-1].item())
|
||||
|
||||
# Better: compute validated host metadata once during request packing and pass it.
|
||||
packed = PackedLayout(
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max(sample_lengths),
|
||||
)
|
||||
for block in self.blocks:
|
||||
hidden = block(hidden, packed_layout=packed)
|
||||
```
|
||||
|
||||
Trace to verify the synchronization disappears. Do not replace a correct
|
||||
dynamic value with a stale constant merely to remove `.item()`.
|
||||
|
||||
### Remove unused masks and allocations
|
||||
|
||||
If packed attention consumes `cu_seqlens` and ignores a dense mask, do not
|
||||
construct the dense mask. Reuse immutable request metadata and preallocated
|
||||
buffers when safe. Prefer direct indexed scatter/gather over constructing large
|
||||
temporary tensors only when duplicate-index semantics and gradients (normally
|
||||
disabled for inference) remain correct.
|
||||
|
||||
Any cross-request cache needs a bounded lifetime and a complete key containing
|
||||
shape, layout, device, dtype, schedule, task, adapter/LoRA, and any value that
|
||||
changes the tensor. Otherwise keep it request-scoped.
|
||||
|
||||
### VAE and reference work
|
||||
|
||||
Profile duplicate VAE decode/postprocess, repeated reference decoding/scans,
|
||||
unnecessary host-device round trips, and full-frame materialization. Apply VAE
|
||||
tiling, slicing, patch parallelism, or on-demand staging only with seam,
|
||||
ordering, color/range, and memory parity tests.
|
||||
|
||||
## USP communication optimization
|
||||
|
||||
“USP works” is not evidence that USP communication is optimized. For Ulysses,
|
||||
Ring, hybrid, and AllGather-KV separately, profile:
|
||||
|
||||
- collective count, bytes, duration, stream, and overlap with compute;
|
||||
- Q/K/V and output all-to-all boundaries;
|
||||
- pack/unpack, padding, metadata broadcast, transpose, and contiguous copies;
|
||||
- head divisibility or advanced uneven-head support;
|
||||
- scaling efficiency from 1 to N devices on the same workload/interconnect;
|
||||
- load balance for unequal packed samples.
|
||||
|
||||
Declare `_sp_plan` split/gather boundaries around meaningful module outputs,
|
||||
not arbitrary Python statements. Keep packed metadata sharded/gathered in the
|
||||
same ownership model as tokens.
|
||||
|
||||
When the target revision offers accelerated Ulysses transport such as
|
||||
`--ulysses-a2a-permute`, qualify it separately from regular Ulysses. Prove the
|
||||
strict scatter-heads/gather-sequence layout selected the fast path, record
|
||||
one-time JIT/readiness cost, prewarm the maximum workspace shape before CUDA
|
||||
graph capture, keep the grow-only workspace on one stream, and exercise cleanup
|
||||
before process-group destruction. Compare collective and E2E deltas after the
|
||||
dense backend and topology are frozen; keep unsupported layouts on an explicit
|
||||
regular-Ulysses fallback.
|
||||
|
||||
For GQA, validate `Q_heads % KV_heads == 0`, pad KV heads first, and derive Q
|
||||
padding from the original ratio. Exercise both contiguous and row-strided QKV
|
||||
layouts; a fast copy path must preserve bitwise values and packed boundaries.
|
||||
|
||||
Cross-attention sometimes uses replicated text K/V while only video/audio Q is
|
||||
sequence-sharded. In that specific case, `skip_sequence_parallel=True` can
|
||||
avoid incorrect or wasteful Q/K/V redistribution:
|
||||
|
||||
```python
|
||||
self.cross_attention = Attention(
|
||||
num_heads=local_heads,
|
||||
num_kv_heads=local_kv_heads,
|
||||
head_size=head_dim,
|
||||
softmax_scale=head_dim**-0.5,
|
||||
causal=False,
|
||||
role="cross",
|
||||
skip_sequence_parallel=True,
|
||||
prefix=f"{prefix}.cross_attn",
|
||||
)
|
||||
```
|
||||
|
||||
Use it only after proving K/V are replicated and Q/output ownership is correct.
|
||||
Otherwise it disables needed communication. Report whether the optimization
|
||||
reduces collective bytes/time and improves E2E scaling.
|
||||
|
||||
## Text encoder and VAE disaggregation
|
||||
|
||||
Disaggregation is a measured architecture decision. First collect:
|
||||
|
||||
| Question | Required measurement |
|
||||
|---|---|
|
||||
| Is the stage large enough? | Encode/decode share at target concurrency and shape |
|
||||
| Is transfer economical? | Tensor/media bytes, serialization, connector latency and bandwidth |
|
||||
| Is there reuse? | Prompt/reference reuse rate, cacheability, fan-out |
|
||||
| Can it scale independently? | Queue saturation and resource utilization per stage |
|
||||
| Is the contract stable? | Shape, dtype, mask/tag/position metadata, ordering |
|
||||
|
||||
### Text encoder candidate
|
||||
|
||||
Define an explicit tensor contract containing hidden states, masks, tags or
|
||||
position IDs, dtype, shape, task/checkpoint revision, and request ID. Support
|
||||
disabling the local encoder so the DiT cannot accidentally recompute it. Test
|
||||
all tasks, prompt lengths, partial TP group membership, retries, backpressure,
|
||||
abort, connector cleanup, and independent replica scaling.
|
||||
|
||||
### VAE candidate
|
||||
|
||||
Separate VAE encode and decode contracts if both exist. Define latent shape,
|
||||
scale/shift, dtype/range, tiling/chunk order, output media ordering, and stage
|
||||
metadata. Measure latent/media transfer against local decode cost. Test tiling
|
||||
seams, RNG/determinism where relevant, chunk ordering, codec/postprocess,
|
||||
backpressure, abort, and worker failure.
|
||||
|
||||
Do not label VAE tiling, patch parallelism, compile, or offload as VAE
|
||||
disaggregation. Likewise, a separate text-encoder TP group is not a separate
|
||||
stage. If no reusable generic stage/connector exists in the target revision,
|
||||
produce a feasibility result or RFC and keep the capability `not tested` rather
|
||||
than inventing a model-local production architecture.
|
||||
|
||||
## Performance acceptance
|
||||
|
||||
For each accepted optimization, require:
|
||||
|
||||
- same fixed workload and correctness artifacts before/after;
|
||||
- actual fast-path execution proven by logs/trace/counters;
|
||||
- local metric plus E2E latency/throughput and memory;
|
||||
- at least three raw fixed-work repetitions after explicit warmup, or a
|
||||
declared arrival-load sample set large enough for every tail percentile;
|
||||
- named hardware, backend, topology, dtype, and task scope;
|
||||
- fallback/rejection tests for unsupported shapes/platforms;
|
||||
- no regression in abort/error cleanup or long-run memory trend.
|
||||
|
||||
Update the feature matrix narrowly. Do not convert a profile-driven direction
|
||||
into a generic “supported” feature merely because a model-specific PR exists.
|
||||
@@ -1,464 +0,0 @@
|
||||
# Production Serving and Validation
|
||||
|
||||
## Contents
|
||||
|
||||
1. [Execution-mode decision](#execution-mode-decision)
|
||||
2. [Request-level batching](#request-level-batching)
|
||||
3. [Step execution and continuous batching](#step-execution-and-continuous-batching)
|
||||
4. [Abort and cleanup](#abort-and-cleanup)
|
||||
5. [Mixed-RPS soak](#mixed-rps-soak)
|
||||
6. [Four CI tracks](#four-ci-tracks)
|
||||
7. [PR evidence checklist](#pr-evidence-checklist)
|
||||
|
||||
## Execution-mode decision
|
||||
|
||||
Request batching and step-wise execution solve different problems:
|
||||
|
||||
| Mode | Pipeline contract | Scheduling/abort boundary |
|
||||
|---|---|---|
|
||||
| Serial request | `forward(DiffusionRequestBatch)` for one request | Whole request; in-denoise preemption is unavailable |
|
||||
| Request batching | `supports_request_batch = True`; one forward consumes compatible independent requests | Whole batch/request |
|
||||
| Single-request step | Full `SupportsStepExecution`; `max_num_seqs=1` | Between denoise steps |
|
||||
| Step continuous batching | Full step contract plus heterogeneous batched-step evidence | Between denoise waves/steps |
|
||||
|
||||
Do not expose four stub methods as an opt-out. The current protocol is
|
||||
structural; if a native pipeline is not step-capable, do not define the four
|
||||
step methods. If it is capable, implement and test all four.
|
||||
|
||||
Treat structural support, a generic request-batch bridge, and model benefit as
|
||||
three separate claims. Before recommending step execution, name a hypothesis
|
||||
that request mode cannot satisfy, define a success threshold, and run a matched
|
||||
request-mode control. If throughput or latency regresses, retain step execution
|
||||
as functional/limited and keep request mode as the production recommendation.
|
||||
|
||||
Choose compatibility keys from every value that changes tensor shapes,
|
||||
semantics, hooks, or scheduler behavior: task, output shape/count, frames,
|
||||
steps/schedule, guidance/CFG, dtype/backend, LoRA/adapter, cache quality policy,
|
||||
and model-specific conditions. Reject or schedule separately rather than
|
||||
silently using the first request's settings.
|
||||
|
||||
## Request-level batching
|
||||
|
||||
A request-batched pipeline returns exactly one `DiffusionOutput` per logical
|
||||
request. This is model-owned pseudocode: `validate_batch_compatibility()` and
|
||||
`_generate_batch()` are placeholders to implement from the official contract,
|
||||
not shared APIs to import.
|
||||
|
||||
```python
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.worker.request_batch import (
|
||||
DiffusionRequestBatch,
|
||||
split_diffusion_output_by_request,
|
||||
)
|
||||
|
||||
class MyPipeline(torch.nn.Module):
|
||||
supports_request_batch = True
|
||||
|
||||
def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]:
|
||||
validate_batch_compatibility(req.sampling_params_list, req.prompts)
|
||||
common = req.sampling_params_list[0]
|
||||
|
||||
text_prompts: list[str] | None = [
|
||||
prompt
|
||||
if isinstance(prompt, str)
|
||||
else (prompt.get("prompt") or "")
|
||||
for prompt in req.prompts
|
||||
]
|
||||
|
||||
prompt_fields = DiffusionRequestBatch.collate_prompt_field_map(
|
||||
req.prompts,
|
||||
{"prompt_embeds": None, "prompt_mask": None},
|
||||
)
|
||||
if prompt_fields["prompt_embeds"] is not None:
|
||||
text_prompts = None
|
||||
generators = req.collate_request_generators(
|
||||
common.num_outputs_per_prompt,
|
||||
default_generator=None,
|
||||
)
|
||||
output = self._generate_batch(
|
||||
prompt=text_prompts,
|
||||
prompt_fields=prompt_fields,
|
||||
generators=generators,
|
||||
sampling=common,
|
||||
)
|
||||
return split_diffusion_output_by_request(
|
||||
DiffusionOutput(output=output),
|
||||
req,
|
||||
num_outputs_per_prompt=common.num_outputs_per_prompt,
|
||||
)
|
||||
```
|
||||
|
||||
The prompt-field collator only handles tensor fields; extract raw text prompts
|
||||
and model-specific negative/alias fields separately, as current in-tree batched
|
||||
pipelines do. Set a raw prompt field to `None` when the matching precomputed
|
||||
embedding is supplied. The exact aliases and generator device depend on the
|
||||
model. Use current collators rather than manual `torch.cat` when they cover the
|
||||
contract; they reject mixed present/missing tensors and incompatible
|
||||
shape/dtype/device.
|
||||
|
||||
Before reading `sampling_params_list[0]`, validate every field that must be
|
||||
common. Tests must cover:
|
||||
|
||||
- two and N requests, independent seeds/generators, one output each;
|
||||
- mixed prompt lengths and packed/padded boundaries;
|
||||
- multiple outputs per prompt and correct result slicing;
|
||||
- compatible versus incompatible shapes, steps, guidance, LoRA, and quality;
|
||||
- one request failure without cross-request output or state corruption;
|
||||
- output ordering and request IDs under different completion times.
|
||||
|
||||
After the model advertises support, candidate serving is:
|
||||
|
||||
```bash
|
||||
vllm serve '<model>' --omni \
|
||||
--max-num-seqs 4 \
|
||||
--request-batch-max-wait-ms 20
|
||||
```
|
||||
|
||||
The wait window trades admission latency for batch formation. Benchmark at 0
|
||||
and candidate nonzero values under the target arrival process.
|
||||
|
||||
## Step execution and continuous batching
|
||||
|
||||
### Full protocol skeleton
|
||||
|
||||
Use the current shared state types and make every mutable value request-local:
|
||||
|
||||
The helpers beginning with `_` and `validate_step_batch()` below are
|
||||
model-owned placeholders; scheduler initialization is scheduler-specific.
|
||||
|
||||
```python
|
||||
import copy
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.models.interface import SupportsStepExecution
|
||||
from vllm_omni.diffusion.worker.input_batch import InputBatch
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
class MyPipeline(torch.nn.Module, SupportsStepExecution):
|
||||
supports_step_execution: ClassVar[bool] = True
|
||||
|
||||
def prepare_encode(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
**kwargs: Any,
|
||||
) -> StepRequestState:
|
||||
prepared = self._prepare_request(state.prompt, state.sampling)
|
||||
state.prompt_embeds = prepared.prompt_embeds
|
||||
state.latents = prepared.latents
|
||||
state.timesteps = prepared.timesteps
|
||||
state.scheduler = copy.deepcopy(self.scheduler)
|
||||
state.scheduler.set_begin_index(0)
|
||||
state.step_index = 0
|
||||
state.extra["task_state"] = prepared.task_state
|
||||
return state
|
||||
|
||||
def denoise_step(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
*,
|
||||
states: Sequence[StepRequestState] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor | None:
|
||||
validate_step_batch(input_batch, states)
|
||||
return self.transformer(**build_batched_denoise_inputs(input_batch))
|
||||
|
||||
def step_scheduler(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
noise_pred: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
timestep = state.current_timestep
|
||||
if timestep is None:
|
||||
raise RuntimeError("step_scheduler called without a current timestep")
|
||||
state.latents = state.scheduler.step(
|
||||
noise_pred,
|
||||
timestep,
|
||||
state.latents,
|
||||
return_dict=False,
|
||||
)[0].to(state.latents.dtype)
|
||||
state.step_index += 1
|
||||
|
||||
def post_decode(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
**kwargs: Any,
|
||||
) -> DiffusionOutput:
|
||||
return DiffusionOutput(output=self.decode(state.latents))
|
||||
```
|
||||
|
||||
This is a shape guide, not drop-in code: verify the target revision's
|
||||
`StepRequestState.extra`, scheduler signature, CFG handling, state fields, and
|
||||
decode contract. Follow a current native pipeline such as Qwen-Image for the
|
||||
exact revision.
|
||||
|
||||
Never use the shared `self.scheduler` to advance a request. Store scheduler,
|
||||
latents, embeddings, timesteps, masks, RNG/generator, task metadata, cache
|
||||
policy, adapter, and output assembly state on `StepRequestState` or its
|
||||
request-local extension. Pipeline fields may hold immutable weights/config and
|
||||
bounded immutable caches only.
|
||||
|
||||
### Qualification sequence
|
||||
|
||||
1. Compare request-mode and step-mode at `max_num_seqs=1` for every task using
|
||||
identical seed/schedule/shape and intermediate latents.
|
||||
2. Abort queued and in-flight requests at several step boundaries.
|
||||
3. Increase to two compatible requests; verify independent scheduler/RNG/state
|
||||
and output order.
|
||||
4. Increase to the target capacity and mix prompt lengths.
|
||||
5. Exercise mixed shape, frames, steps/schedule, task, guidance, and quality.
|
||||
Batch only compatible rows; otherwise split/reject predictably.
|
||||
6. Inject a failure in encode, denoise, scheduler, and decode independently.
|
||||
7. Run the production soak and fault matrix.
|
||||
|
||||
Candidate commands only after capability validation:
|
||||
|
||||
```bash
|
||||
# Conservative step execution and step-boundary abort.
|
||||
vllm serve '<model>' --omni \
|
||||
--step-execution \
|
||||
--max-num-seqs 1
|
||||
|
||||
# Experimental batched-step candidate after multi-request E2E.
|
||||
vllm serve '<model>' --omni \
|
||||
--step-execution \
|
||||
--max-num-seqs 4
|
||||
```
|
||||
|
||||
If an attention backend is only correct for single-request step mode, fail
|
||||
early or keep the recipe at `max_num_seqs=1`. Do not describe generic runtime
|
||||
support as model support.
|
||||
|
||||
## Abort and cleanup
|
||||
|
||||
### Required semantics
|
||||
|
||||
For queued and in-flight abort/disconnect:
|
||||
|
||||
- emit exactly one terminal aborted/error result;
|
||||
- do not run `post_decode` after abort unless a documented partial-output mode
|
||||
explicitly requires it;
|
||||
- do not increment `step_index` or apply a scheduler result after cancellation;
|
||||
- remove scheduler/state cache entries and request IDs;
|
||||
- release latents, prompt/reference tensors, cache hooks, DLO buffers, connector
|
||||
messages, uploaded/temp files, and output buffers;
|
||||
- keep other requests alive;
|
||||
- make cleanup idempotent when disconnect, timeout, and worker error race;
|
||||
- drop late results/exceptions for already-resolved or cancelled futures
|
||||
without terminating the shared result-pump thread;
|
||||
- complete the next valid request with its requested cache/offload policy.
|
||||
|
||||
Test cancellation before engine admission, while queued, after encode, during
|
||||
early/middle/final denoise steps, before decode, and while returning/persisting
|
||||
output. Include async job cancellation and client socket disconnect.
|
||||
|
||||
### Failure injection matrix
|
||||
|
||||
| Location | Injection | Assertions |
|
||||
|---|---|---|
|
||||
| Validation/upload | bad MIME, count/bytes overflow, truncated media | 400/413, no job/engine submit, no temp leak |
|
||||
| Load/start | missing fused shard, unsupported topology | startup fails, actionable error, no silent init |
|
||||
| Encode | exception/OOM/timeout | one error, request state/temp released |
|
||||
| Denoise | exception/OOM/worker exit | peers handled, hooks/buffers released, health behavior explicit |
|
||||
| Scheduler | invalid state/shape | no latent/step partial commit |
|
||||
| Decode/post | exception/codec error | no corrupt successful result, partial output removed |
|
||||
| Result/IPC | late result after cancel, large split tensor, orphan SHM | pump remains alive, one terminal result, exact ownership cleanup |
|
||||
| Connector/disagg | timeout/retry/duplicate | idempotency, backpressure, no orphan payload |
|
||||
|
||||
After each injection, check server health and issue a known-good request. If
|
||||
the process must restart, document and test the restart policy; do not claim
|
||||
transparent recovery.
|
||||
|
||||
### Large-output transport
|
||||
|
||||
Treat worker-to-engine IPC and engine-to-client transport as separate hops.
|
||||
For batched video, prove that each logical output does not accidentally retain
|
||||
or serialize the complete batch storage. Exercise the target revision's shared
|
||||
memory or artifact-handle path above and below its threshold, including
|
||||
non-contiguous views, multiple outputs, consumer failure, timeout, abort, and
|
||||
worker exit. Verify every shared segment/file is released exactly once.
|
||||
|
||||
Freeze the media contract at each hop: dtype, value range, shape/layout,
|
||||
contiguity, request ownership, encoding/color model, payload bytes, and pending
|
||||
consumers. Device-side float-to-uint8 preparation, D2H/IPC, planar/interleaved
|
||||
conversion, CPU encode/mux, and delivery off the asyncio event loop are
|
||||
different optimizations. Validate their route selection and fallbacks
|
||||
independently. Byte-identical HTTP MP4 output does not prove that a raw offline
|
||||
tensor kept its dtype or range.
|
||||
|
||||
For long-window output, validate segment ordering and audio/video continuity,
|
||||
then exercise payloads around serializer, shared-memory, HTTP, and codec size
|
||||
limits. Chunking must reassemble exactly, reject malformed/missing chunks, and
|
||||
release partial state after abort, timeout, or peer failure.
|
||||
|
||||
Qualify chunk production, transport, and public streaming as separate layers.
|
||||
A VAE callback must define owned representation, frame offsets, final marker,
|
||||
rank ownership, source/config gate, and collective-safe failure, but it does not
|
||||
prove SHM/ZMQ delivery, bounded backpressure, cancellation, codec reuse, or
|
||||
client streaming. Measure time to first owned chunk, first encoded fragment,
|
||||
steady cadence, mux finalization, and complete client artifact independently.
|
||||
|
||||
Measure remote MP4 encoding, serialization/copies, network transfer, and client
|
||||
materialization separately from denoise/decode. An inference-kernel speedup does
|
||||
not establish serving throughput if output transport dominates or leaks.
|
||||
|
||||
## Mixed-RPS soak
|
||||
|
||||
Use a realistic request mix and arrival distribution. Include small/large
|
||||
shapes, short/long prompts, all advertised tasks, cache tiers, and a small rate
|
||||
of abort/disconnect/error injections. Run phases below, near, and above measured
|
||||
saturation so queue/backpressure behavior is visible.
|
||||
|
||||
Example serving benchmark phase:
|
||||
|
||||
```bash
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--base-url http://127.0.0.1:8091 \
|
||||
--endpoint /v1/videos \
|
||||
--model '<model>' \
|
||||
--dataset trace \
|
||||
--dataset-path /path/to/versioned-production-trace.jsonl \
|
||||
--task t2v \
|
||||
--num-prompts 500 \
|
||||
--request-rate 0.5 \
|
||||
--max-concurrency 8 \
|
||||
--warmup-requests 8 \
|
||||
--warmup-concurrency 8 \
|
||||
--output-file soak-near-saturation.json
|
||||
```
|
||||
|
||||
Use a task-appropriate trace/media dataset. Run the same phase at rates selected
|
||||
from an initial saturation curve; a literal `0.5` is illustrative only.
|
||||
|
||||
On supported GPU hosts, wrap a test or soak with the repository monitor:
|
||||
|
||||
```bash
|
||||
bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \
|
||||
pytest -s -v tests/dfx/stability/scripts/test_stability_<model>.py -m slow
|
||||
```
|
||||
|
||||
Check the target revision's monitor backends. Do not claim CPU/NPU monitoring
|
||||
from a reserved CLI choice unless collection is implemented and verified.
|
||||
|
||||
Report:
|
||||
|
||||
- offered and achieved RPS, success/error/abort rates;
|
||||
- p50/p95/p99 and queue/service/stage times, with the successful sample count
|
||||
and arrival process used for each percentile;
|
||||
- throughput per device and batch/wave utilization;
|
||||
- per-rank HBM and process-tree PSS trend/slope;
|
||||
- cache size/hit rate, temp-file/disk growth, open FDs/handles;
|
||||
- outstanding shared-memory/artifact handles and encoded/network bytes;
|
||||
- worker/engine health, restarts, OOM/timeouts, backpressure response;
|
||||
- total duration, request count/mix, raw logs/JSON/monitor artifact hashes.
|
||||
|
||||
A short burst or repeated serial curl loop is not a stability test.
|
||||
|
||||
## Four CI tracks
|
||||
|
||||
Keep the tracks independent so a failure answers a specific question. These
|
||||
four readiness tracks are not the repository's numbered CI levels. Map them
|
||||
onto the current L1-L5 taxonomy:
|
||||
|
||||
| Readiness track | Repository CI placement |
|
||||
|---|---|
|
||||
| Function | L1 unit/logic plus L2 basic online/offline E2E; extend real-model scenarios in L3/L4 |
|
||||
| Accuracy | L3 real-model gates for important cases; deeper and broader nightly coverage in L4 |
|
||||
| Performance | Important thresholds in L3 where they fit the time budget; full baselines/regression jobs in nightly L4 |
|
||||
| Reliability | Cheap rejection/recovery assertions can run earlier, but long stability, fault injection, and reliability suites belong to weekly L5 |
|
||||
|
||||
Distributed topology is a test case within the appropriate level, not a
|
||||
replacement definition for a level.
|
||||
|
||||
### 1. Function CI
|
||||
|
||||
Cover:
|
||||
|
||||
- strict unquantized and advertised quantized loading;
|
||||
- fused source-shard completeness and prefix routing;
|
||||
- official task positive boundaries and neighboring negative cases;
|
||||
- offline, sync, async normalization/parity;
|
||||
- single-device reference plus every advertised topology smoke;
|
||||
- output type/count/order and request ID isolation;
|
||||
- feature rejection paths that must fail early.
|
||||
|
||||
### 2. Accuracy CI
|
||||
|
||||
Pin official implementation/checkpoint/vLLM-Omni revisions, prompts and media
|
||||
hashes, seed, scheduler/sigma/steps, dimensions/frames/FPS, guidance, dtype,
|
||||
backend, and topology. Compare at suitable levels:
|
||||
|
||||
- component outputs or selected blocks;
|
||||
- one or more intermediate denoise latents;
|
||||
- final image/video/audio artifact metrics;
|
||||
- temporal consistency and audio/video synchronization where relevant.
|
||||
|
||||
Set tolerance per modality/hardware/backend with rationale. Keep failure
|
||||
artifacts and metadata. A single visual inspection does not define CI.
|
||||
|
||||
For quantization trajectory tooling, inspect the target revision's current
|
||||
`vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py` and
|
||||
reuse it when compatible instead of creating an ad hoc comparator.
|
||||
|
||||
### 3. Performance CI
|
||||
|
||||
Pin workload and best deployment. Include a resident recommended row and a
|
||||
small-HBM/DLO row when both are production targets. Declare:
|
||||
|
||||
- one explicit warmup exclusion and at least three measured fixed-work
|
||||
repetitions;
|
||||
- raw result JSON and environment identity;
|
||||
- median/mean plus range for small fixed-work A/Bs; p50/p95/p99 only for an
|
||||
arrival-load run with enough samples, plus throughput/RPS, stage time,
|
||||
HBM/PSS;
|
||||
- regression metric, threshold, baseline owner, and update procedure;
|
||||
- correctness/accuracy check linked to the same configuration.
|
||||
|
||||
Use the current harness config style, for example:
|
||||
|
||||
```bash
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
|
||||
--test-config-file tests/dfx/perf/tests/test_<model>_vllm_omni.json
|
||||
```
|
||||
|
||||
Verify parser options on the target revision. Do not copy a JSON schema from a
|
||||
different model without matching its endpoint, task, dataset, and deploy path.
|
||||
|
||||
### 4. Reliability CI
|
||||
|
||||
Run a bounded CI soak plus a longer scheduled soak. Cover:
|
||||
|
||||
- below/near/above-saturation arrival phases;
|
||||
- queued/in-flight abort and disconnect;
|
||||
- OOM/timeout and worker/process failure appropriate to the platform;
|
||||
- DLO/cache/connector cleanup;
|
||||
- health, fast-fail/backpressure, restart/recovery policy;
|
||||
- memory/temp/FD trend and successful known-good request after each fault.
|
||||
|
||||
Keep reliability thresholds independent from latency thresholds. A fast server
|
||||
that leaks memory or hangs after cancellation fails production readiness.
|
||||
|
||||
## PR evidence checklist
|
||||
|
||||
The PR description must include or link:
|
||||
|
||||
- official source/checkpoint revisions and API/limit matrix;
|
||||
- scoped feature compatibility matrix with the four allowed states;
|
||||
- implementation notes for FP8, DLO, request cache, attention/fusion, batching;
|
||||
- exact environment, deploy/serve, every task curl, benchmark and soak commands;
|
||||
- fixed accuracy inputs and artifacts;
|
||||
- raw before/after performance runs and memory traces;
|
||||
- output-contract manifests and stage accounting from device preparation
|
||||
through the complete client artifact;
|
||||
- abort/disconnect/fault results and cleanup evidence;
|
||||
- Function/Accuracy/Performance/Reliability CI files and ownership;
|
||||
- limitations/TODOs that remain `not tested` rather than implied support.
|
||||
|
||||
Separate the Day-0 milestone from production readiness. If the PR only closes
|
||||
some gates, title and description should say which scoped gates it closes.
|
||||
@@ -1,103 +0,0 @@
|
||||
---
|
||||
name: quantization
|
||||
description: Work on vLLM-Omni quantization for diffusion, autoregressive, omni, or multi-stage models. Use when choosing or adding methods such as fp8, int8, gguf, mxfp8, mxfp4, mxfp4_dualscale, ModelOpt, AutoRound, INC, msModelSlim, awq, or gptq; debugging quantized loading; or validating memory, speed, and output quality.
|
||||
---
|
||||
|
||||
# vLLM-Omni Quantization
|
||||
|
||||
Use this skill for quantization work in `vllm-omni`. Start from the local
|
||||
code and docs, then pick the smallest validated path for the model, method,
|
||||
hardware, and quality target.
|
||||
|
||||
## First Checks
|
||||
|
||||
Before changing code or recommending a command, identify:
|
||||
|
||||
- Model and task: AR text, omni/TTS, text-to-image, text-to-video, or multi-stage diffusion.
|
||||
- Hardware backend: CUDA generation, Ascend NPU, Intel XPU, or another platform.
|
||||
- Quantization mode: online from BF16/FP16 weights, pre-quantized checkpoint, runtime KV/FA quantization, or per-component routing.
|
||||
- Scope: transformer/DiT, thinker language model, text encoder, VAE, audio/vision encoder, talker, or a specific stage.
|
||||
- Validation target: memory reduction, latency/throughput, output quality, or all three.
|
||||
|
||||
Do not assume a method is supported just because the CLI accepts the string.
|
||||
Check `docs/user_guide/quantization/` and the closest implementation first.
|
||||
|
||||
## Quick Decision
|
||||
|
||||
| Task | Start With |
|
||||
|------|------------|
|
||||
| Choose a method or command | `references/methods.md` and `references/modality-compat.md` |
|
||||
| Use `build_quant_config()` or per-component routing | `references/methods.md` |
|
||||
| Work on diffusion quantization | `references/diffusion.md` |
|
||||
| Add quantization to a new model | `references/adding-models.md` |
|
||||
| Convert or load ModelOpt FP8 checkpoints | `references/modelopt-fp8.md` |
|
||||
| Debug ModelOpt, GGUF, AutoRound, MXFP, or serialized Int8 loading | `references/diffusion.md` and method docs under `docs/user_guide/quantization/` |
|
||||
|
||||
## Current Runtime Shape
|
||||
|
||||
The unified entrypoint is:
|
||||
|
||||
```python
|
||||
from vllm_omni.quantization import build_quant_config
|
||||
```
|
||||
|
||||
It supports method strings, flat method dictionaries, per-component
|
||||
dictionaries, existing `QuantizationConfig` objects, and `None`. The factory
|
||||
delegates generic methods to upstream `vllm` and keeps vLLM-Omni overrides for
|
||||
diffusion or omni-specific routing:
|
||||
|
||||
- `gguf`
|
||||
- `int8`
|
||||
- `mxfp8`
|
||||
- `mxfp4`
|
||||
- `mxfp4_dualscale`
|
||||
- `inc`, `auto-round`, `auto_round`
|
||||
- ModelOpt auto-detected configs: `modelopt`, `modelopt_fp4`, `modelopt_mixed`
|
||||
|
||||
Use `ComponentQuantizationConfig` when only one stage or component should be
|
||||
quantized. Pre-quantized ModelOpt-style checkpoints should not spill into
|
||||
vision/audio encoders that have no corresponding scale tensors.
|
||||
|
||||
## Ownership Boundary
|
||||
|
||||
- Upstream `vllm` owns generic quantization configs, kernels, loader semantics,
|
||||
hardware capability rules, and generic AR quantization methods.
|
||||
- `vllm-omni` owns unified config routing, diffusion-specific wrappers,
|
||||
component scoping, GGUF or ModelOpt checkpoint adapters, model-specific
|
||||
prefix mapping, docs, examples, and validation.
|
||||
|
||||
If a new method needs missing generic kernels or loader behavior, fix upstream
|
||||
`vllm` first. In `vllm-omni`, add thin integration and model wiring.
|
||||
|
||||
## Working Loop
|
||||
|
||||
1. Read the local method doc under `docs/user_guide/quantization/`.
|
||||
2. Inspect the active factory and config class in `vllm_omni/quantization/`.
|
||||
3. Inspect the target model pipeline and transformer for stable prefixes and
|
||||
whether every relevant vLLM linear layer receives `quant_config`.
|
||||
4. For pre-quantized checkpoints, inspect `transformer/config.json` and the
|
||||
checkpoint tensor names before running full generation.
|
||||
5. Validate with the same prompt, seed, size, scheduler, steps, dtype, eager or
|
||||
non-eager mode, and parallelism as the BF16 baseline.
|
||||
6. Report quality, latency, and memory separately. Do not call a method
|
||||
supported until quality is checked.
|
||||
|
||||
## Common Mistakes
|
||||
|
||||
| Symptom | Likely Cause | Fix |
|
||||
|---------|--------------|-----|
|
||||
| `--quantization` has no visible effect | Wrong scope or unsupported model path | Check component routing and method docs |
|
||||
| Some layers stay BF16 unexpectedly | `quant_config` was not threaded into all vLLM linear layers | Audit transformer constructors and prefixes |
|
||||
| Quality collapses but loading succeeds | Too many sensitive layers were quantized | Add model-specific `ignored_layers` and compare to BF16 |
|
||||
| ModelOpt checkpoint loads but output is corrupted | Prefixes, packed-module mapping, scale routing, or BF16 fallback is wrong | Use `references/modelopt-fp8.md` |
|
||||
| GGUF shape or tensor mismatch | Missing architecture-specific adapter | Add explicit adapter mapping; avoid generic fallback |
|
||||
| Online and offline MXFP4 disagree | Wrong `mxfp4` vs `mxfp4_dualscale` mode | Check `docs/user_guide/quantization/mxfp4.md` |
|
||||
| Quantized path is slower than BF16 | Kernel path, compile mode, dtype, or shape mismatch | Compare eager-to-eager and non-eager-to-non-eager |
|
||||
|
||||
## References
|
||||
|
||||
- General methods and config routing: [references/methods.md](references/methods.md)
|
||||
- Modality and model scope: [references/modality-compat.md](references/modality-compat.md)
|
||||
- Diffusion implementation patterns: [references/diffusion.md](references/diffusion.md)
|
||||
- Adding model support: [references/adding-models.md](references/adding-models.md)
|
||||
- ModelOpt FP8 conversion and loading: [references/modelopt-fp8.md](references/modelopt-fp8.md)
|
||||
@@ -1,70 +0,0 @@
|
||||
# Adding Quantization Support to a Model
|
||||
|
||||
Use this checklist when extending quantization to a new architecture or stage.
|
||||
|
||||
## Classify Ownership
|
||||
|
||||
| Case | Owner |
|
||||
|------|-------|
|
||||
| New generic AR method, kernel, or loader semantics | upstream `vllm` first |
|
||||
| Existing upstream method needs diffusion wiring | `vllm-omni` model and config integration |
|
||||
| GGUF or ModelOpt tensor mapping for a diffusion architecture | `vllm-omni` adapter |
|
||||
| NPU MXFP or msModelSlim diffusion path | `vllm-omni` platform/method wrapper and docs |
|
||||
| Multi-stage omni checkpoint quantizes only the thinker LM | `vllm-omni` component scoping |
|
||||
|
||||
Avoid building a private quantization stack in model code.
|
||||
|
||||
## Model Wiring Checklist
|
||||
|
||||
1. Find the closest supported model implementation.
|
||||
2. Confirm the model uses vLLM linear layers that accept `quant_config`.
|
||||
3. Add `quant_config` to the transformer constructor.
|
||||
4. Pass stable `prefix` values through every linear layer.
|
||||
5. Handle fused modules and packed mappings explicitly.
|
||||
6. Ensure `load_weights()` and any `WeightsMapper` remap `ignored_layers` or
|
||||
`modules_to_not_convert`.
|
||||
7. Keep tokenizer, scheduler, VAE, text encoder, audio/vision encoders, and
|
||||
talker BF16 unless method-specific docs validate them.
|
||||
8. Add or update a method doc under `docs/user_guide/quantization/`.
|
||||
|
||||
## Checkpoint Adapter Checklist
|
||||
|
||||
For pre-quantized checkpoints:
|
||||
|
||||
- Inspect `transformer/config.json` first.
|
||||
- Check `quant_method`, `method`, `producer.name`, `quant_algo`, serialized
|
||||
flags, `ignored_layers`, and any method-specific fields.
|
||||
- Verify tensor name mapping before generation.
|
||||
- Verify packed modules such as `to_qkv`, `add_kv_proj`, and `w13`.
|
||||
- Confirm scale tensor names and shapes.
|
||||
- Allow dequantization back to BF16 for intentionally skipped target layers.
|
||||
|
||||
## Sensitive Layer Discovery
|
||||
|
||||
Start conservative. Quantize the backbone first and keep output-side or routing
|
||||
layers BF16 until quality is proven.
|
||||
|
||||
Frequently sensitive areas:
|
||||
|
||||
- attention output projections such as `to_out`
|
||||
- FFN down projections such as `w2`
|
||||
- final projections such as `proj_out`, `final_layer`, and output heads
|
||||
- modulation, timestep, position, image, or text embedders
|
||||
- refiner blocks
|
||||
- MoE routers, experts selected by routing, and layernorm-adjacent modules
|
||||
- VAE, text/vision towers, audio encoders, and codec/talker stages
|
||||
|
||||
Do not reuse one model family's ignore list blindly.
|
||||
|
||||
## Tests and Evidence
|
||||
|
||||
Minimum evidence for a PR:
|
||||
|
||||
- loader/config test or smoke command
|
||||
- fixed-seed BF16 vs quantized output comparison
|
||||
- quality metrics and saved outputs for precision-changing work
|
||||
- memory and latency comparison under identical execution mode
|
||||
- failure behavior for unsupported models or missing checkpoint fields
|
||||
|
||||
When the PR adds a method to a support table, include the exact model,
|
||||
hardware, command, quality threshold, and artifact path used for validation.
|
||||
@@ -1,113 +0,0 @@
|
||||
# Diffusion Quantization Reference
|
||||
|
||||
Use this file for DiT quantization, config routing, checkpoint adapters, and
|
||||
quality or loader debugging.
|
||||
|
||||
## Supported Method Families
|
||||
|
||||
The current local diffusion quantization families are:
|
||||
|
||||
- CUDA/GPU online or checkpoint: `fp8`, `int8`, `gguf`, ModelOpt
|
||||
- Ascend NPU: `int8`, `mxfp8`, `mxfp4`, `mxfp4_dualscale`, msModelSlim paths
|
||||
- Intel XPU: AutoRound/INC and online MXFP8 paths where documented
|
||||
|
||||
Always cross-check the method doc before claiming model support. The CLI
|
||||
accepting a method string is not enough.
|
||||
|
||||
## Implementation Starting Points
|
||||
|
||||
| Area | Files |
|
||||
|------|-------|
|
||||
| Unified factory | `vllm_omni/quantization/factory.py` |
|
||||
| Per-component routing | `vllm_omni/quantization/component_config.py` |
|
||||
| GGUF diffusion method | `vllm_omni/quantization/gguf_config.py` |
|
||||
| Int8 diffusion method | `vllm_omni/quantization/int8_config.py` |
|
||||
| MXFP8 diffusion method | `vllm_omni/quantization/mxfp8_config.py` |
|
||||
| MXFP4 and dual-scale method | `vllm_omni/quantization/mxfp4_config.py` |
|
||||
| AutoRound/INC omni extension | `vllm_omni/quantization/inc_config.py` |
|
||||
| ModelOpt checkpoint adapters | `vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py` |
|
||||
| FP8 scaled-mm runtime patches | `vllm_omni/patch.py` |
|
||||
| Similarity tool | `vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py` |
|
||||
|
||||
## Threading `quant_config`
|
||||
|
||||
For a diffusion transformer:
|
||||
|
||||
1. Accept `quant_config: QuantizationConfig | None` in the transformer.
|
||||
2. Pass `quant_config` and stable `prefix` values to every vLLM linear layer.
|
||||
3. Preserve packed-module mappings for fused QKV, FFN, or model-specific fused
|
||||
projections.
|
||||
4. Call `apply_vllm_mapper()` on configs that need HF-to-vLLM name remapping.
|
||||
5. For component configs, keep prefixes specific enough to avoid quantizing VAE
|
||||
or encoders by accident.
|
||||
|
||||
Common skipped areas are `proj_out`, output projections, modulation layers,
|
||||
embedders, refiner blocks, MoE routers, and layernorm-adjacent paths.
|
||||
|
||||
## Online vs Offline
|
||||
|
||||
Online quantization:
|
||||
|
||||
- starts from BF16/FP16 checkpoint weights
|
||||
- computes or stores quantized weights at load time
|
||||
- typically uses `--quantization <method>` or `quantization="<method>"`
|
||||
|
||||
Offline or pre-quantized checkpoints:
|
||||
|
||||
- include quantized weights and scales on disk
|
||||
- usually carry `quantization_config` in `transformer/config.json`
|
||||
- should auto-detect when the config is complete
|
||||
- may require model-specific adapter logic for tensor names and packed modules
|
||||
|
||||
Do not mix online flags with incompatible offline checkpoint configs. If the
|
||||
active config and disk config differ, `resolve_quant_config_from_disk()` should
|
||||
raise or rebuild rather than silently loading corrupted weights.
|
||||
|
||||
## GGUF Rules
|
||||
|
||||
- Require `quantization_config.gguf_model`.
|
||||
- Use the normal base model for tokenizer, scheduler, text encoder, and VAE.
|
||||
- Add explicit architecture adapters for tensor-name mapping.
|
||||
- Use `named_parameters()` and `named_buffers()`, not `state_dict()`, to
|
||||
discover loadable names.
|
||||
- Guard fused QKV/KV rewrites so tensors are not rewritten twice.
|
||||
- Flatten N-D activations around the GGUF matmul and restore shape.
|
||||
|
||||
Unsupported architectures should fail clearly instead of falling back to a fake
|
||||
generic mapper.
|
||||
|
||||
## MXFP Rules
|
||||
|
||||
- `mxfp8` is W8A8 microscaling, currently centered on NPU and documented XPU
|
||||
behavior.
|
||||
- `mxfp4` is online single-scale W4A4 on NPU.
|
||||
- `mxfp4_dualscale` is online/offline dual-scale W4A4 with BF16 fallback; the
|
||||
offline path is the production path when calibrated `mul_scale` exists.
|
||||
- Offline dual-scale checkpoints must inject per-transformer `ignored_layers`
|
||||
into config because cascade transformers can have different BF16 fallback
|
||||
sets.
|
||||
|
||||
Do not load an offline dual-scale checkpoint with the online single-scale path.
|
||||
|
||||
## Common Failures
|
||||
|
||||
| Symptom | Likely Cause | Fix |
|
||||
|---------|--------------|-----|
|
||||
| `Unknown quantization method` | Factory override or upstream registration is missing | Add the right override or upstream method |
|
||||
| Missing scale tensors | Wrong scope, wrong adapter, or BF16 layer not allowed to dequantize | Check config scope and adapter scale routing |
|
||||
| Output rank collapses from 3-D to 2-D | FP8 scaled-mm kernel ignored output shape | Check the patch in `vllm_omni/patch.py` |
|
||||
| Quality degrades silently | Sensitive layers quantized | Add `ignored_layers`; compare to BF16 |
|
||||
| Only one cascade transformer works | Disk config differs per transformer but active config is reused | Rebuild from each transformer's config |
|
||||
| Quantized path slower than BF16 | Kernel, dtype, compile, or benchmark mode mismatch | Compare eager-to-eager and non-eager-to-non-eager |
|
||||
|
||||
## Verification
|
||||
|
||||
For a new or changed diffusion quantization path, run:
|
||||
|
||||
1. Config and loader smoke test.
|
||||
2. One fixed-seed generation against BF16.
|
||||
3. Numeric comparison with the similarity tool or equivalent metrics.
|
||||
4. Serving test if the path is user-facing online.
|
||||
5. Memory and latency check under the same execution mode as the baseline.
|
||||
|
||||
Keep output artifacts when precision changes are part of the PR.
|
||||
@@ -1,82 +0,0 @@
|
||||
# General Quantization Methods
|
||||
|
||||
Use this file for method selection and unified `vllm-omni` config syntax. For
|
||||
diffusion-specific wiring, see `diffusion.md`. For ModelOpt checkpoint export
|
||||
and loading details, see `modelopt-fp8.md`.
|
||||
|
||||
## Unified Entry Point
|
||||
|
||||
`vllm_omni.quantization.build_quant_config()` accepts:
|
||||
|
||||
- `None` or `"none"` to disable quantization
|
||||
- a method string such as `"fp8"`, `"int8"`, `"mxfp8"`, `"awq"`, or `"auto-round"`
|
||||
- a flat dict with `"method"` or `"quant_method"`
|
||||
- a per-component dict such as `{"transformer": {"method": "fp8"}, "vae": None}`
|
||||
- an already-built `QuantizationConfig`
|
||||
|
||||
Per-component dicts are routed by `ComponentQuantizationConfig` using
|
||||
longest-prefix matching. Prefix stability matters: if a model remaps HF names
|
||||
to vLLM names, apply the mapper to `ignored_layers` or component prefixes.
|
||||
|
||||
## Local Overrides
|
||||
|
||||
The local factory delegates most methods to upstream `vllm`, but has vLLM-Omni
|
||||
overrides for methods that need diffusion, NPU, XPU, or component behavior:
|
||||
|
||||
| Method | Local Config | Main Use |
|
||||
|--------|--------------|----------|
|
||||
| `gguf` | `DiffusionGGUFConfig` | Pre-quantized diffusion transformer weights |
|
||||
| `int8` | `DiffusionInt8Config` | Online or serialized W8A8 diffusion transformers |
|
||||
| `mxfp8` | `DiffusionMXFP8Config` | W8A8 MXFP8 diffusion on NPU and online XPU path |
|
||||
| `mxfp4` | `DiffusionMXFP4Config` | Online W4A4 MXFP4 diffusion on NPU |
|
||||
| `mxfp4_dualscale` | `DiffusionMXFP4DualScaleMixedConfig` | Online/offline W4A4 dual-scale plus BF16 fallback on NPU |
|
||||
| `inc`, `auto-round`, `auto_round` | `OmniINCConfig` | AutoRound/INC checkpoint loading, including MXFP8 checkpoint metadata |
|
||||
| ModelOpt config dicts | upstream ModelOpt configs through auto-detect | FP8, NVFP4, or mixed ModelOpt checkpoints |
|
||||
|
||||
Check `SUPPORTED_QUANTIZATION_METHODS` in `vllm_omni/quantization/factory.py`
|
||||
before adding a new method name.
|
||||
|
||||
## AR and Generic Quantization
|
||||
|
||||
For AR-backed models, upstream `vllm` methods usually apply:
|
||||
|
||||
- `awq` and `gptq`: 4-bit or 8-bit weight quantization with calibration,
|
||||
commonly used for text and AR stages.
|
||||
- `fp8`: FP8 weight path or KV-cache FP8 depending on flags and hardware.
|
||||
- `inc` or `auto-round`: pre-quantized checkpoints with Intel Neural
|
||||
Compressor / AutoRound metadata.
|
||||
|
||||
Use these for the AR language model stage only when the checkpoint and model
|
||||
guide support it. Omni and TTS models often keep audio, vision, talker, and
|
||||
codec stages in BF16.
|
||||
|
||||
## Diffusion and Offline Methods
|
||||
|
||||
For diffusion transformers, prefer the method-specific docs:
|
||||
|
||||
- `docs/user_guide/quantization/fp8.md`
|
||||
- `docs/user_guide/quantization/int8.md`
|
||||
- `docs/user_guide/quantization/gguf.md`
|
||||
- `docs/user_guide/quantization/modelopt.md`
|
||||
- `docs/user_guide/quantization/mxfp8.md`
|
||||
- `docs/user_guide/quantization/mxfp4.md`
|
||||
- `docs/user_guide/quantization/autoround.md`
|
||||
- `docs/user_guide/quantization/msmodelslim.md`
|
||||
|
||||
Pre-quantized checkpoints should normally auto-detect from config. Runtime
|
||||
flags such as `--force-cutlass-fp8`, `--linear-backend cutlass`, and
|
||||
`--moe-backend cutlass` select kernels; they do not quantize a BF16 checkpoint.
|
||||
|
||||
## Validation
|
||||
|
||||
Use fixed-seed A/B comparisons and record:
|
||||
|
||||
- model path and quantized checkpoint path
|
||||
- method config and `ignored_layers`
|
||||
- prompt, size, frame count, steps, scheduler, dtype, and seed
|
||||
- eager/non-eager mode, compile flags, and parallelism
|
||||
- output metrics such as PSNR, MAE, cosine similarity, SSIM, LPIPS, plus visual artifacts
|
||||
- latency and peak memory, reported separately from quality
|
||||
|
||||
The helper `vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity`
|
||||
is the default lightweight image/video comparison tool for diffusion paths.
|
||||
@@ -1,47 +0,0 @@
|
||||
# Quantization Compatibility by Modality
|
||||
|
||||
Use this as a routing guide. The source of truth is the local method docs under
|
||||
`docs/user_guide/quantization/` and the current model implementation.
|
||||
|
||||
## Summary
|
||||
|
||||
- AR and omni language-model quantization mostly follows upstream `vllm`, with
|
||||
vLLM-Omni scoping for multi-stage models.
|
||||
- Diffusion quantization is method-specific and model-specific. It usually
|
||||
targets the diffusion transformer only.
|
||||
- Pre-quantized checkpoints should quantize only the component that has
|
||||
quantized weights and scale tensors.
|
||||
|
||||
## Model Type Matrix
|
||||
|
||||
| Model Type | Typical Quantization Scope | Notes |
|
||||
|------------|----------------------------|-------|
|
||||
| AR text models | Language model | Upstream `vllm` methods usually apply |
|
||||
| Omni/TTS models | Thinker or AR language-model stage | Audio, vision, talker, and codec stages usually stay BF16 |
|
||||
| DiT image models | Diffusion transformer | Check FP8, Int8, GGUF, ModelOpt, AutoRound docs |
|
||||
| DiT video models | Diffusion transformer | HunyuanVideo FP8 and Wan2.2 MXFP paths are documented; validate other video paths |
|
||||
| Multi-stage diffusion | Selected stage only | Use per-component routing and per-stage validation |
|
||||
| Audio diffusion | Usually unsupported | Prefer dtype, offload, cache, or parallelism unless docs say otherwise |
|
||||
|
||||
## Method Hints
|
||||
|
||||
| Goal | Candidate |
|
||||
|------|-----------|
|
||||
| AR memory reduction | AWQ, GPTQ, AutoRound, ModelOpt where checkpoint supports it |
|
||||
| CUDA diffusion W8A8 | FP8 or Int8 |
|
||||
| Pre-quantized CUDA diffusion checkpoint | ModelOpt or GGUF |
|
||||
| Ascend NPU diffusion | Int8, MXFP8, MXFP4, MXFP4 DualScale, msModelSlim |
|
||||
| Intel XPU checkpoint | AutoRound/INC paths |
|
||||
| Conservative image quality | selective FP8/Int8 with `ignored_layers` |
|
||||
| Unsupported diffusion model | cache acceleration, CPU/layerwise offload, dtype, TP/SP/CFG parallelism |
|
||||
|
||||
## Validation Rule
|
||||
|
||||
Compatibility means more than successful loading. A method is compatible only
|
||||
after it has:
|
||||
|
||||
- correct component scope
|
||||
- complete checkpoint/config metadata if offline
|
||||
- fixed-seed quality comparison against BF16
|
||||
- acceptable latency and memory behavior
|
||||
- docs or examples that state model, method, hardware, and caveats
|
||||
@@ -1,122 +0,0 @@
|
||||
# ModelOpt FP8 Conversion and Loading
|
||||
|
||||
Use this when exporting or loading ModelOpt FP8 checkpoints for diffusion
|
||||
transformers. This reference captures the practical conversion lessons from the
|
||||
2026-04-21 ModelOpt FP8 paste and aligns them with the current
|
||||
`vllm-omni` ModelOpt adapter.
|
||||
|
||||
## Core Rule
|
||||
|
||||
Treat ModelOpt FP8 as selective checkpoint quantization, not one-click
|
||||
whole-model FP8. Quantize the main transformer or DiT backbone, keep sensitive
|
||||
or peripheral modules BF16, serialize a complete ModelOpt checkpoint, and let
|
||||
vLLM-Omni auto-detect it from config.
|
||||
|
||||
Do not pass `--quantization fp8` to force a ModelOpt checkpoint path. The
|
||||
checkpoint's `quantization_config` should select `modelopt`.
|
||||
|
||||
## Conversion Strategy
|
||||
|
||||
Prefer:
|
||||
|
||||
- backbone attention Q/K/V and FFN expansion projections
|
||||
- conservative per-model ignore lists
|
||||
- MHA quantizers disabled until basic linear quantization is stable
|
||||
- full module-name prefixes in ignore/routing logic
|
||||
- BF16 fallback for output-side and routing-sensitive layers
|
||||
|
||||
Avoid quantizing by default:
|
||||
|
||||
- VAE
|
||||
- text, vision, or audio encoders
|
||||
- embedders and modulation layers
|
||||
- final heads and output projections
|
||||
- refiner modules
|
||||
- MoE routers and layernorm-sensitive blocks
|
||||
|
||||
## Model-Specific Starting Points
|
||||
|
||||
| Model Family | Quantize First | Keep BF16 First |
|
||||
|--------------|----------------|-----------------|
|
||||
| Z-Image | attention q/k/v, FFN w1/w3 | `to_out`, `w2`, `noise_refiner`, `context_refiner`, embedders, modulation, final/output |
|
||||
| Qwen-Image | transformer backbone linear layers | `proj_out`, `img_in`, `txt_in`, norms, modulation, position/time/embed layers |
|
||||
| FLUX2-dev / FLUX2-klein | backbone linear layers | `proj_out`, context/x embedders, `norm_out`, stream modulation, time embeddings |
|
||||
| HunyuanImage-3.0 | `model.model` backbone | VAE, vision tower, token/time/image embedders, final image head, LM head, routers, layernorm-sensitive blocks |
|
||||
|
||||
These are starting points, not support claims. Validate with BF16 outputs.
|
||||
|
||||
## Required Checkpoint Metadata
|
||||
|
||||
`transformer/config.json` must include a complete `quantization_config`.
|
||||
At minimum it should identify ModelOpt and the algorithm:
|
||||
|
||||
```json
|
||||
{
|
||||
"quantization_config": {
|
||||
"producer": {"name": "modelopt"},
|
||||
"quant_method": "modelopt",
|
||||
"quant_algo": "FP8"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Current vLLM-Omni auto-detects:
|
||||
|
||||
- `FP8` and `FP8_PER_CHANNEL_PER_TOKEN` -> `modelopt`
|
||||
- `NVFP4` -> `modelopt_fp4`
|
||||
- `MIXED_PRECISION` -> `modelopt_mixed`
|
||||
|
||||
Packed modules and scales must map correctly:
|
||||
|
||||
- `to_qkv` from `to_q`, `to_k`, `to_v`
|
||||
- `add_kv_proj` from `add_q_proj`, `add_k_proj`, `add_v_proj`
|
||||
- `w13` from `w1`, `w3`
|
||||
- `.input_scale`, `.weight_scale`, `.weight_scale_2`, `.weight_scale_inv`
|
||||
|
||||
For selective FP8, target layers that remain BF16 must be allowed to
|
||||
dequantize from FP8 weights using the matching `weight_scale`, or the adapter
|
||||
must skip the quantized scale tensors for that BF16 target.
|
||||
|
||||
## Runtime Signals
|
||||
|
||||
Healthy loading should show ModelOpt auto-detection and a ModelOpt linear
|
||||
kernel path. Useful signals include:
|
||||
|
||||
- auto-detected quantization from config
|
||||
- selected ModelOpt FP8 linear method
|
||||
- CUTLASS or FlashInfer FP8 scaled-mm backend when enabled and supported
|
||||
- non-eager path compiling the transformer when expected
|
||||
|
||||
If online serving fails while offline inference works, inspect warmup and
|
||||
serving initialization before changing checkpoint conversion.
|
||||
|
||||
## Validation Order
|
||||
|
||||
1. Check config completeness.
|
||||
2. Run an export checker such as `examples/quantization/check_modelopt_fp8_export.py`.
|
||||
3. Confirm actual FP8 tensors and expected disk-size reduction.
|
||||
4. Run one offline generation with fixed prompt, seed, size, and steps.
|
||||
5. Run serving with `vllm serve --omni` and verify `/v1/images/generations`.
|
||||
6. Compare against BF16 using visual review plus metrics such as LPIPS, PSNR,
|
||||
MAE, cosine similarity, and SSIM.
|
||||
7. Compare performance only under matched modes:
|
||||
- BF16 eager vs FP8 eager
|
||||
- BF16 non-eager vs FP8 non-eager
|
||||
|
||||
FP8 is not automatically faster. Check whether execution reaches FP8 kernels,
|
||||
whether compile mode is comparable, and whether the workload shape fits the
|
||||
kernel path.
|
||||
|
||||
## Triage
|
||||
|
||||
| Symptom | First Check |
|
||||
|---------|-------------|
|
||||
| Startup failure | `config.json`, `model_index.json`, `quantization_config`, missing scales |
|
||||
| Loads but output is corrupted | Over-quantized sensitive layers or wrong full-name prefixes |
|
||||
| Offline works but online fails | Dummy warmup, serving init, compile path |
|
||||
| FP8 is not faster | Kernel selection, dtype, compile mode, workload shape |
|
||||
| BF16 fallback fails | Missing `weight_scale` for dequantization or wrong packed-module mapping |
|
||||
|
||||
The success criterion is stable, correctly scoped checkpoint loading with
|
||||
quality preserved enough for the target use case, not maximizing the number of
|
||||
FP8 layers.
|
||||
@@ -1,67 +0,0 @@
|
||||
# Repository Skills for vLLM-Omni
|
||||
|
||||
This directory contains repository-scale skills maintained for `vllm-omni`.
|
||||
They capture repeatable workflows for common contributor and maintainer tasks
|
||||
such as model integration, CI-aligned testing, performance optimization, and
|
||||
pull request review.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
Each skill lives in its own directory under `.claude/skills/`. A skill may
|
||||
include:
|
||||
|
||||
- `SKILL.md`: the main workflow and operating instructions
|
||||
- `references/`: focused reference material used by the skill
|
||||
- `scripts/`: small helper scripts used by the skill
|
||||
|
||||
## Using the Skills
|
||||
|
||||
Coding agents that discover repository skills can invoke the relevant skill by
|
||||
name. For other agents, point them to the skill's `SKILL.md` and ask them to
|
||||
read it before changing code. Combine a domain skill with `vllm-omni-test` when
|
||||
tests or CI coverage change, then use `precheck-pr` for the contributor's final
|
||||
self-check.
|
||||
|
||||
## Available Skills
|
||||
|
||||
- [`add-diffusion-model`](add-diffusion-model/SKILL.md): guides integration of
|
||||
a new diffusion model into `vllm-omni`
|
||||
- [`add-tts-model`](add-tts-model/SKILL.md): covers integration of new TTS
|
||||
models and related serving workflows
|
||||
- [`diffusion-perf-opt`](diffusion-perf-opt/SKILL.md): guides diffusion model
|
||||
performance optimization, including profiling traces, parallel strategies,
|
||||
stage timing analysis, and benchmark-driven tuning
|
||||
- [`find-simplifications`](find-simplifications/SKILL.md): finds evidence-backed
|
||||
opportunities to remove or merge dead, duplicated, speculative,
|
||||
over-generalized, or unnecessarily defensive vLLM-Omni code
|
||||
- [`precheck-pr`](precheck-pr/SKILL.md): self-checks a branch before creating a
|
||||
PR by validating title format, dead code, simplification opportunities,
|
||||
accuracy and performance claims, and merge readiness
|
||||
- [`production-add-diffusion-model`](production-add-diffusion-model/SKILL.md):
|
||||
takes a working Day-0 diffusion integration through API/limit parity,
|
||||
feature-combination evidence, quantization/offload, performance, production
|
||||
serving, hardware recipes, and CI readiness
|
||||
- [`quantization`](quantization/SKILL.md): guides quantization method selection,
|
||||
model integration, checkpoint loading, and quality/performance validation
|
||||
for vLLM-Omni
|
||||
- [`review-pr`](review-pr/SKILL.md): provides a frozen-snapshot, contract-aware
|
||||
workflow for maintainers and reviewers; contributors should use
|
||||
`precheck-pr` for self-review
|
||||
- [`vllm-omni-npu-model-runner-upgrade`](vllm-omni-npu-upgrade/SKILL.md):
|
||||
upgrades NPU model runners to align with the latest vllm-ascend
|
||||
`NPUModelRunner`
|
||||
- [`vllm-omni-test`](vllm-omni-test/SKILL.md): guides generation and execution
|
||||
of CI-aligned tests (L1–L4), pytest marker selection (`core_model` /
|
||||
`advanced_model` / `full_model`, `omni` / `tts` / `diffusion`), Buildkite
|
||||
wiring (`test-ready.yml`, `test-merge.yml`, `test-nightly.yml`,
|
||||
`test-weekly.yml`), and copy-paste local plus CI-like `pytest` commands; see
|
||||
`references/test-routing.md` for level-to-command mapping
|
||||
|
||||
## Maintenance Guidelines
|
||||
|
||||
- Keep skill names short and task-oriented.
|
||||
- Prefer repository-local paths, commands, and examples.
|
||||
- Avoid hardcoding fast-changing support matrices unless the skill is actively
|
||||
maintained alongside those changes.
|
||||
- Treat skills as contributor tooling: optimize for clarity, actionability, and
|
||||
low maintenance overhead.
|
||||
@@ -1,247 +0,0 @@
|
||||
---
|
||||
name: review-pr
|
||||
description: Review pull requests and local branches for vllm-project/vllm-omni with a frozen snapshot, module-design ownership, feature-design overlays, targeted validation, and concise evidence-backed findings. Use for default, detailed, or repeat maintainer reviews; checking correctness, compatibility, tests, benchmarks, model additions, distributed changes, or breaking behavior; and identifying or explicitly requesting the most relevant code-owner reviewers. Use precheck-pr instead for an author's pre-submit self-check.
|
||||
---
|
||||
|
||||
# Review vLLM-Omni Pull Requests
|
||||
|
||||
Review like a maintainer: direct, selective, and focused on issues that CI does
|
||||
not prove. Prefer a few high-confidence findings over exhaustive commentary.
|
||||
Zero findings is a valid result.
|
||||
|
||||
## Quality contract
|
||||
|
||||
Make every finding:
|
||||
|
||||
- **Correct:** prove a reachable failure, not a suspicion.
|
||||
- **Prioritized:** lead with merge blockers and high-impact defects.
|
||||
- **Actionable:** identify the smallest safe fix direction.
|
||||
- **Evidence-based:** cite code, tests, docs, CI, or measurements.
|
||||
- **Concise:** avoid review templates and repeated summaries.
|
||||
- **Calibrated:** match severity to user and maintainer impact.
|
||||
|
||||
Do not report unrelated backlog, style already enforced by pre-commit, or a
|
||||
missing test that would not protect changed behavior. Do report new allowlist
|
||||
or budget entries in `check_forbidden_imports.py` / `check_torch_cuda.py` /
|
||||
`check_tts_adapter.py` / `check_buildkite.py` unless the PR justifies them:
|
||||
those are policy changes, not lint noise.
|
||||
|
||||
## Select the input and depth
|
||||
|
||||
Use `vllm-project/vllm-omni` as the base repository. Accept its forks and local
|
||||
checkouts; use another skill for unrelated repositories.
|
||||
|
||||
| Input | Review surface |
|
||||
| --- | --- |
|
||||
| PR number or URL | Frozen PR metadata, full diff, and relevant threads. |
|
||||
| Local branch/worktree | Frozen target-base SHA through committed, staged, unstaged, and in-scope untracked changes. |
|
||||
| Pre-filled context | Reuse supplied metadata; fetch only missing facts and the full diff. |
|
||||
|
||||
Default to maintainer brevity. A detailed or audit request expands coverage and
|
||||
lists `path:line` findings, but keeps the same confidence and severity bar.
|
||||
|
||||
## Reference guide
|
||||
|
||||
Load references in review order: process, one primary module contract, matching
|
||||
feature designs, evidence checks, then delivery. Every file is linked directly
|
||||
below; do not load unrelated module references.
|
||||
|
||||
Each concise reference links to the maintained
|
||||
[vLLM-Omni documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/).
|
||||
For branch-specific behavior, inspect the matching `docs/` file in the reviewed
|
||||
checkout first; use the published latest docs for current guidance and discovery.
|
||||
If docs and live code disagree, verify the code/tests and report the drift.
|
||||
|
||||
### Review process
|
||||
|
||||
| Reference | Read when |
|
||||
| --- | --- |
|
||||
| [review-execution.md](references/process/review-execution.md) | Every review; freeze inputs, inspect safely, and deliver against the same snapshot. |
|
||||
| [general-checks.md](references/process/general-checks.md) | Every review; apply repository-wide correctness and evidence rules. |
|
||||
| [design-contracts.md](references/process/design-contracts.md) | Every production review; resolve branch-local module and feature design status. |
|
||||
| [review-routing.md](references/process/review-routing.md) | After the diff census; select one primary module and conditional overlays. |
|
||||
|
||||
### Primary module contract
|
||||
|
||||
| Reference | Read when |
|
||||
| --- | --- |
|
||||
| [entrypoints.md](references/modules/entrypoints.md) | Offline/CLI/API ingress, validation, rendering, streaming, or sessions change. |
|
||||
| [configuration.md](references/modules/configuration.md) | Config construction, deploy/stage schema, defaults, registry, or topology changes. |
|
||||
| [input-output-modality.md](references/modules/input-output-modality.md) | Requests, messages, serialization, output types, accumulation, or completion change. |
|
||||
| [error-contracts.md](references/modules/error-contracts.md) | Error classification, fatality, propagation, sanitization, or rendering changes. |
|
||||
| [engine-orchestration.md](references/modules/engine-orchestration.md) | Cross-stage routing, request state, output ordering, RPC correlation, or terminal convergence changes. |
|
||||
| [stage-runtime.md](references/modules/stage-runtime.md) | Placement, startup, readiness, replica identity, affinity, membership, or shutdown changes. |
|
||||
| [omni-connector.md](references/modules/omni-connector.md) | Cross-stage/process/device/node transport or synchronization changes. |
|
||||
| [model-integration.md](references/modules/model-integration.md) | Registration, preprocessing, loading, runners, or model-specific execution changes. |
|
||||
| [ar-runtime.md](references/modules/ar-runtime.md) | AR scheduling, request/cache state, adapters, workers, or upstream vLLM semantics change. |
|
||||
| [diffusion.md](references/modules/diffusion.md) | Diffusion runtime, models, batching, parallelism, or offload changes. |
|
||||
| [execution-platforms.md](references/modules/execution-platforms.md) | Hardware selection, capabilities, vendor workers, kernels, or patches change. |
|
||||
| [cache-management.md](references/modules/cache-management.md) | Cache identity, reuse, validity, reset, eviction, or teardown changes. |
|
||||
| [quantization.md](references/modules/quantization.md) | Quantization selection, checkpoint metadata, layer mapping, precision, or constraints change. |
|
||||
| [observability.md](references/modules/observability.md) | Metrics, logs, units, labels, correlation, or lifecycle changes. |
|
||||
| [profiling.md](references/modules/profiling.md) | Profiling instrumentation, traces, start/stop lifecycle, or overhead changes. |
|
||||
| [benchmarking.md](references/modules/benchmarking.md) | Benchmark workload, metric calculation, CLI, or result metadata changes. |
|
||||
|
||||
### Feature-design overlays
|
||||
|
||||
| Reference | Read when |
|
||||
| --- | --- |
|
||||
| [runtime-stage-execution.md](references/features/runtime-stage-execution.md) | Disaggregated inference, async chunk/output/materialization, or prefix caching changes. |
|
||||
| [communication.md](references/features/communication.md) | A concrete OmniConnector backend or its deployment contract changes. |
|
||||
| [diffusion-acceleration.md](references/features/diffusion-acceleration.md) | Diffusion parallelism, attention, quantization, cache, batching, or offload changes. |
|
||||
| [infrastructure-performance.md](references/features/infrastructure-performance.md) | Metrics infrastructure or documented speech optimization stacks change. |
|
||||
|
||||
### Evidence and quality checks
|
||||
|
||||
| Reference | Read when |
|
||||
| --- | --- |
|
||||
| [model-addition-checklist.md](references/checks/model-addition-checklist.md) | A model, architecture, loader, processor, registry, pipeline config, or deploy config is added. |
|
||||
| [perf-verification.md](references/checks/perf-verification.md) | The PR makes a latency, throughput, memory, or quality claim. |
|
||||
| [test-quality-evaluation.md](references/checks/test-quality-evaluation.md) | Tests change, are absent for risky code, or may not exercise production behavior. |
|
||||
| [tests-docs-checklist.md](references/checks/tests-docs-checklist.md) | Coverage, CI markers, examples, user docs, or PR evidence need review. |
|
||||
| [verification.md](references/checks/verification.md) | Hardware, a server, or a runnable affected path is available for active verification. |
|
||||
| [examples-policy.md](../precheck-pr/references/examples-policy.md) | The PR adds, copies, or renames Python under `examples/`; apply the canonical policy shared with `precheck-pr`. |
|
||||
| [find-simplifications](../find-simplifications/SKILL.md) | Every review; run a diff-scoped subtraction and simplification pass after correctness blockers. |
|
||||
|
||||
### Delivery and reviewer coordination
|
||||
|
||||
| Reference | Read when |
|
||||
| --- | --- |
|
||||
| [maintainer-style-study.md](references/delivery/maintainer-style-study.md) | Findings are ready for concise maintainer-style delivery. |
|
||||
| [review-requests.md](references/delivery/review-requests.md) | The user asks to identify, suggest, request, or ping code-owner reviewers. |
|
||||
|
||||
## Workflow
|
||||
|
||||
### 1. Freeze and report the snapshot
|
||||
|
||||
Pin the base and head before reading source or running validation. Within 60 seconds,
|
||||
report the pinned head, CI, mergeability, and preliminary findings in the host
|
||||
conversation. Do not wait for CI or post this update to GitHub.
|
||||
|
||||
If the target changes while fetching, discard the evidence and retry once. If
|
||||
it changes again, report the churn and wait for a stable target.
|
||||
|
||||
For a trusted PR head, materialize the pinned head in an isolated detached
|
||||
worktree. A worktree freezes identity but is not a security sandbox. Treat fork
|
||||
heads as untrusted unless the user and environment policy explicitly establish
|
||||
otherwise: execute them only in a disposable, secret-free sandbox with restricted
|
||||
filesystem, network, and resources; without one, use static SHA-addressed reads
|
||||
and CI evidence only. For a local review, freeze the committed, index, worktree,
|
||||
and NUL-safe in-scope untracked contents. Follow
|
||||
[review-execution.md](references/process/review-execution.md) for trust gates,
|
||||
state fingerprints, and byte-for-byte staleness checks.
|
||||
|
||||
### 2. Build the diff census
|
||||
|
||||
Group files into production code, tests, docs, configuration, build/CI, and
|
||||
generated artifacts. Map each changed production file and test group to the PR
|
||||
goal. Compare the title/body claims with the actual diff; use linked issues only
|
||||
when they define the contract or reproduction.
|
||||
|
||||
Mark unrelated scope and unexplained generated artifacts. Do not infer behavior
|
||||
from the PR description without tracing the live code.
|
||||
|
||||
### 3. Route from the live behavior
|
||||
|
||||
Trace each claimed behavior through the changed producer to its live consumer,
|
||||
then use [design-contracts.md](references/process/design-contracts.md) and
|
||||
[review-routing.md](references/process/review-routing.md) to select one primary
|
||||
module contract, a second only for a real documented cross-boundary call path,
|
||||
and every matching feature-design and evidence overlay. Treat titles and paths
|
||||
as hints; live behavior and the frozen head's current design metadata are
|
||||
authoritative. For docs-, tests-, or CI-only changes, route to the production
|
||||
contract they protect or use only the applicable evidence checks.
|
||||
|
||||
### 4. Run the blocker scan
|
||||
|
||||
Apply every category in [general-checks.md](references/process/general-checks.md) before
|
||||
lower-priority comments.
|
||||
|
||||
If the diff census contains an added, copied, or renamed Python path under
|
||||
`examples/`, read and apply the canonical
|
||||
[examples policy](../precheck-pr/references/examples-policy.md). Treat a new
|
||||
model-specific Python example as blocking. Do not flag model-specific example
|
||||
debt that the PR only modifies or removes, and do not run the rest of the
|
||||
author-oriented `precheck-pr` workflow.
|
||||
|
||||
For each changed value or behavior, trace:
|
||||
|
||||
```text
|
||||
public ingress -> validation/defaulting -> producer -> transformations
|
||||
-> stage/worker/connector boundary -> final consumer -> terminal cleanup
|
||||
```
|
||||
|
||||
Cover every applicable offline/online, streaming/non-streaming, sync/async,
|
||||
feature-on/off, topology, and compatibility path. Search bounded callers and
|
||||
sibling implementations rather than assuming the changed hunk is the only path.
|
||||
|
||||
### 5. Apply module and feature contracts
|
||||
|
||||
Apply the reference set selected in step 3 and any matching repo-local skill.
|
||||
Read the exact module and feature pages in the frozen head, including status,
|
||||
ownership boundary, dependencies, candidate invariants, safe-change guide, and
|
||||
promotion gate. Candidate or draft rules are questions, not blockers, unless
|
||||
current code, tests, or policy enforce them. Inspect both sides of any config,
|
||||
registry, serialization, connector, cache, or stage boundary.
|
||||
|
||||
Read and apply [find-simplifications](../find-simplifications/SKILL.md) on every
|
||||
review. Constrain it to the diff and the adjacent ownership, callers, or
|
||||
consumers needed to prove a candidate. Check whether added or expanded helpers,
|
||||
classes, state, fallback and compatibility branches, data movement, or public
|
||||
behavior can be deleted, merged, moved, or inlined. Zero candidates is a valid
|
||||
result. Do not widen the review into repository backlog or report speculative
|
||||
style preferences as simplification findings.
|
||||
|
||||
### 6. Verify the changed path
|
||||
|
||||
Before each validation group, verify the frozen SHA plus the tracked, index,
|
||||
untracked, and ignored-file fingerprint, or recreate a pristine snapshot. On a
|
||||
trusted head or inside the required sandbox, run an import/version preflight,
|
||||
then the narrowest relevant tests and low-cost static checks. Bind every result
|
||||
to the head SHA, snapshot fingerprint, and environment fingerprint. Never run
|
||||
imports, tests, builds, hooks, or repo-configurable tooling from an untrusted
|
||||
head on the reviewer host.
|
||||
|
||||
- Treat CI as status evidence; inspect only the first overlapping failure.
|
||||
- For docs-only changes, use diff hygiene, links/build checks, and bounded live
|
||||
contract verification instead of dependency setup or pytest.
|
||||
- For hardware-dependent paths, run available static/CPU checks and name the
|
||||
exact GPU/NPU gap. Never simulate device evidence.
|
||||
- For performance or accuracy claims, require comparable base/head runs with
|
||||
the same environment, workload, warmup, repetitions, and quality criteria.
|
||||
|
||||
Stop when each changed semantic path has a supported finding or an explicit
|
||||
no-issue conclusion. Do not search further only to increase confidence.
|
||||
|
||||
### 7. Consolidate and deliver
|
||||
|
||||
Verify each finding against the current diff, deduplicate by root cause, and
|
||||
order by severity.
|
||||
|
||||
Re-read the remote head and reverify or recreate the pristine validation
|
||||
snapshot immediately before delivery. If either changed, mark the review stale
|
||||
and restart from the new snapshot.
|
||||
|
||||
Return findings first. Use
|
||||
[maintainer-style-study.md](references/delivery/maintainer-style-study.md) to keep them
|
||||
direct and brief. Each finding must include an exact `path:line`, trigger or
|
||||
call path, current behavior, impact, and smallest fix direction. If there are
|
||||
no findings, say so briefly and name material validation gaps.
|
||||
|
||||
Keep the review read-only unless the user explicitly authorizes posting. Do not
|
||||
submit `APPROVE`, `COMMENT`, or `REQUEST_CHANGES`, add labels, edit code, or push
|
||||
commits as an implied part of review.
|
||||
|
||||
### 8. Optionally request focused owner reviews
|
||||
|
||||
Only when the user asks to identify or request reviewers, read
|
||||
[review-requests.md](references/delivery/review-requests.md). Rank path-matched
|
||||
CODEOWNERS with the frozen module page's owners or required reviewers and
|
||||
documented governance expertise; propose one to three focused reviewers with an
|
||||
explicit contract rationale.
|
||||
|
||||
Identifying or suggesting reviewers is read-only. Requesting reviewers or
|
||||
posting `@mention` comments changes external state and requires explicit user
|
||||
authorization. When authorized, recheck the head, deduplicate existing
|
||||
requests, and post at most one consolidated comment. Do not infer this
|
||||
permission from a request to review the code.
|
||||
@@ -1,4 +0,0 @@
|
||||
interface:
|
||||
display_name: "vLLM-Omni PR Review"
|
||||
short_description: "Review vLLM-Omni PRs and route owner requests"
|
||||
default_prompt: "Use $review-pr to review this vLLM-Omni pull request with owner-specific rules and evidence-backed findings."
|
||||
@@ -1,121 +0,0 @@
|
||||
# Model Addition Checklist
|
||||
|
||||
Read this reference when a PR adds or expands a model architecture, pipeline,
|
||||
loader, processor, registry entry, pipeline config, or deploy config.
|
||||
|
||||
Official docs: [model contribution guides](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/),
|
||||
[adding an omni model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_omni_model/),
|
||||
[adding a diffusion model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_diffusion_model/),
|
||||
[adding a TTS model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_tts_model/),
|
||||
[supported models](https://docs.vllm.ai/projects/vllm-omni/en/latest/models/supported_models/),
|
||||
[diffusion features](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion_features/),
|
||||
and [feature compatibility](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/feature_compatibility/).
|
||||
|
||||
## Match claims to the diff
|
||||
|
||||
- Reconcile every claimed architecture, modality, variant, feature, and example
|
||||
with changed files and registry entries.
|
||||
- Resolve config model types, architecture names, callable paths, exports, and
|
||||
dependency extras against live symbols.
|
||||
- Check sibling generations sharing an architecture name are disambiguated by
|
||||
a stable config predicate rather than import order or an accidental default.
|
||||
|
||||
## Close the integration chain
|
||||
|
||||
Trace:
|
||||
|
||||
```text
|
||||
public model id -> config/pipeline selection -> registry -> loader/processor
|
||||
-> stage inputs -> model execution -> stage/public output
|
||||
```
|
||||
|
||||
Verify optional dependencies fail with an actionable message, weight names and
|
||||
dtypes map correctly, and each advertised serving mode reaches the production
|
||||
dispatcher. Confirm outputs are non-empty and valid for their modality, shape,
|
||||
sample rate, or response schema.
|
||||
|
||||
## Keep support documentation truthful
|
||||
|
||||
- Build an evidence-backed inventory of the new model's exact architecture and
|
||||
checkpoint identifiers, modalities and tasks, hardware backends, serving
|
||||
modes, and supported acceleration, parallelism, cache, offload, and
|
||||
quantization features. Treat untested support as unknown, not supported.
|
||||
- Update `docs/models/supported_models.md` with the architecture, model family,
|
||||
example checkpoints, and only the backends validated by the PR.
|
||||
- For a diffusion model, add it to the applicable ImageGen, VideoGen, or
|
||||
AudioGen table in `docs/user_guide/diffusion_features.md`. Fill every feature
|
||||
column according to its legend and document partial support or constraints.
|
||||
- Update `docs/user_guide/feature_compatibility.md` when the model changes valid
|
||||
feature combinations, configuration constraints, examples, or limitations,
|
||||
and update any other branch-local support table affected by the claims.
|
||||
- Cross-check every documented support mark against the registry, configuration,
|
||||
production-path tests, and PR evidence; do not copy a sibling model's support
|
||||
claims without validation.
|
||||
|
||||
## Ship the model recipe
|
||||
|
||||
- Require every new model PR to add or update one model-family recipe under
|
||||
`recipes/<vendor>/` and its row in the `recipes/README.md` Available Recipes
|
||||
table. Follow [`recipes/TEMPLATE.md`](../../../../../recipes/TEMPLATE.md) and
|
||||
the layout rules in
|
||||
[`recipes/README.md`](../../../../../recipes/README.md); the
|
||||
external recipes repository is a structural reference, not a substitute.
|
||||
- Include only tested task, serving-mode, and hardware configurations. Keep the
|
||||
exact model identifiers, flags, commands, verification, feature support,
|
||||
constraints, and limitations consistent with code, tests, examples, and the
|
||||
support documentation.
|
||||
- Cover each validated platform in its own hardware section. Use one recipe per
|
||||
model family by default; justify a split when configurations cannot be kept
|
||||
clear in one file, and do not add placeholder sections for untested platforms.
|
||||
|
||||
## Compare the reference implementation
|
||||
|
||||
Require every new model PR to include a head-versus-reference accuracy or
|
||||
quality and performance comparison:
|
||||
|
||||
- Name the canonical implementation and pin its repository revision, package
|
||||
version, configuration, and checkpoint. If it cannot run, explain why and use
|
||||
the closest runnable reference; do not silently omit the comparison.
|
||||
- Match inputs or dataset, seed, output settings, precision or quantization,
|
||||
hardware, software, warmup, repetitions, synchronization, and timing and
|
||||
memory scope. Include exact commands and environment details for both sides.
|
||||
- Compare accuracy or output quality with a task-appropriate metric and explicit
|
||||
tolerance. When no reliable metric exists, provide paired fixed-input outputs,
|
||||
artifacts, and evaluation criteria, and explain expected numerical or
|
||||
qualitative differences.
|
||||
- Report end-to-end latency, task-appropriate throughput, peak device memory,
|
||||
run-to-run variability, and head/reference ratios at equivalent output quality.
|
||||
- Prefer a timing split for preprocessing or encoding, each model stage,
|
||||
inter-stage transfer or communication, decoding or postprocessing, and output
|
||||
assembly. Define the boundaries, account for overlap, reconcile the split with
|
||||
end-to-end time, and explain any unavailable stage measurement.
|
||||
|
||||
Use [perf-verification.md](perf-verification.md) for the comparison protocol and
|
||||
repository benchmark and profiling links.
|
||||
|
||||
## Remove accidental surface
|
||||
|
||||
Search bounded call sites for:
|
||||
|
||||
- inference-dead training `forward()` paths, unused factories/wrappers, never-set
|
||||
branch keys, and immediately discarded parameters;
|
||||
- duplicate payload strings, validation, or shape coercion across stages;
|
||||
- compatibility aliases in brand-new code without an existing caller;
|
||||
- private symbols re-exported as public API or module-level side effects.
|
||||
|
||||
Keep code only when it has a distinct live caller, invariant, or compatibility
|
||||
contract. Prefer one typed producer-consumer schema to repeated string keys.
|
||||
|
||||
## Require proportionate evidence
|
||||
|
||||
- Run a representative production-path inference and assert output content, not
|
||||
only process survival.
|
||||
- Require profiling or A/B tables when the PR makes performance, memory,
|
||||
precision, or quality claims, or when a suspected hot-path/device bug needs
|
||||
that evidence. Do not impose fixed utilization or regression thresholds that
|
||||
the repository does not define.
|
||||
- Require focused registry/config/loading tests, a representative E2E path, and
|
||||
user-facing model/usage documentation.
|
||||
|
||||
Use [diffusion.md](../modules/diffusion.md) for diffusion models and
|
||||
[perf-verification.md](perf-verification.md) for quantitative claims.
|
||||
@@ -1,51 +0,0 @@
|
||||
# Performance and Accuracy Verification
|
||||
|
||||
Activate this reference when the PR adds a model or claims or intentionally
|
||||
changes latency, throughput, memory, scaling, precision, or output quality.
|
||||
|
||||
Official docs: [profiling](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/profiling/),
|
||||
[serving benchmarks](https://docs.vllm.ai/projects/vllm-omni/en/latest/cli/bench/serve/),
|
||||
and [metrics](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/metrics/).
|
||||
|
||||
## Comparable A/B contract
|
||||
|
||||
Compare frozen base and head for an existing path. For a new model absent from
|
||||
base, compare head with a pinned canonical reference implementation. Use the same:
|
||||
|
||||
- hardware, driver, software versions, model/checkpoint, and dependencies;
|
||||
- input or dataset, seed, precision, batch/concurrency, topology, and feature
|
||||
flags;
|
||||
- warmup, measured repetitions, synchronization, timing scope, and memory
|
||||
collection method.
|
||||
|
||||
Keep exact commands and report variability rather than a cherry-picked run.
|
||||
Use the repository's benchmark when it covers the claim; otherwise use the
|
||||
smallest reproducible workload that reaches the changed production path.
|
||||
|
||||
## Evidence table
|
||||
|
||||
| Dimension | Typical evidence |
|
||||
| --- | --- |
|
||||
| Latency | End-to-end latency and, when available, TTFT plus a defined per-stage and transfer-time split. |
|
||||
| Throughput/scaling | Requests, tokens, frames, or audio duration per second across the claimed concurrency/topology. |
|
||||
| Memory | Peak allocated/reserved device memory and OOM boundary if relevant. |
|
||||
| Quality/accuracy | Repository metric or known-good output comparison with an explicit tolerance; paired samples when no metric exists. |
|
||||
|
||||
Every optimization needs correctness or quality evidence. A faster result on a
|
||||
different workload, precision, seed, or topology is not a valid comparison.
|
||||
|
||||
## Reviewer verification ladder
|
||||
|
||||
1. Run base/head A/B on suitable hardware when affordable.
|
||||
2. If only one side can run, verify the script and request a comparable pair.
|
||||
3. If hardware or assets are unavailable, audit methodology, code path, and
|
||||
contributor evidence; name the exact unverified claim.
|
||||
|
||||
Classify discrepancies before reporting: implementation regression, benchmark
|
||||
bug, environmental drift, noise, or unsupported claim. Explain material
|
||||
regressions against the PR's stated goal or repository contract; do not invent
|
||||
universal percentage thresholds.
|
||||
|
||||
Report claimed and measured values together, bind them to the frozen SHAs and
|
||||
environment, and keep unavailable hardware as a validation gap rather than a
|
||||
fabricated pass.
|
||||
@@ -1,39 +0,0 @@
|
||||
# Test Quality Evaluation
|
||||
|
||||
Read this reference when tests change, a risky production path has no test, or
|
||||
a test-only PR may not prove the intended behavior.
|
||||
|
||||
Official docs: [test system overview](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_system_overview/)
|
||||
and [test writing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_writing_guide/).
|
||||
|
||||
## Static proof check
|
||||
|
||||
For each changed semantic path, ask whether the test would fail if that behavior
|
||||
were reverted or broken. Check:
|
||||
|
||||
- assertions pin the contract, not only non-null output, logs, or “no crash”;
|
||||
- the production dispatcher, registry, connector, or scheduler is reached
|
||||
instead of the changed behavior being mocked away;
|
||||
- fakes preserve relevant types, MRO, shapes, devices, async behavior, and
|
||||
lifecycle transitions;
|
||||
- seeds, ordering, timing, synchronization, external services, and numeric
|
||||
tolerances are deterministic or explicitly controlled;
|
||||
- normal, invalid, boundary, feature-off, failure/cancellation, and regression
|
||||
paths are covered where the diff changes them;
|
||||
- run-level and domain markers place hardware/model tests in the intended CI
|
||||
lane without silently skipping the contract.
|
||||
|
||||
Map source symbols to tests with bounded `rg` searches; do not assume the test
|
||||
tree mirrors production paths.
|
||||
|
||||
## Runtime check
|
||||
|
||||
Run the narrowest affected tests supported by the environment. Record skipped
|
||||
hardware/model cases as gaps. Classify failures as code, test, infrastructure,
|
||||
or flaky before turning them into findings; one passing rerun does not erase a
|
||||
flakiness signal.
|
||||
|
||||
Keep grades or matrices internal. Report only concrete code bugs or the one or
|
||||
two test defects that leave changed behavior materially unprotected. Use
|
||||
[tests-docs-checklist.md](tests-docs-checklist.md) for CI and documentation
|
||||
coverage.
|
||||
@@ -1,48 +0,0 @@
|
||||
# Tests and Documentation Checklist
|
||||
|
||||
Use this reference to connect changed behavior to coverage, CI selection,
|
||||
examples, documentation, and reproducible PR evidence.
|
||||
|
||||
Official docs: [test writing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_writing_guide/),
|
||||
[test execution guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_execution_guide/),
|
||||
and [documentation guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/DOCS_GUIDE/).
|
||||
|
||||
## Coverage packet
|
||||
|
||||
For each high-risk change, record a compact internal packet:
|
||||
|
||||
```text
|
||||
change and failure risk -> existing unit/E2E coverage -> uncovered boundary
|
||||
-> smallest test and stable assertion that closes the gap
|
||||
```
|
||||
|
||||
Prefer a regression test that fails on the frozen base for bug fixes. Check
|
||||
numeric tolerances, deterministic inputs, realistic mocks, hardware/domain
|
||||
markers, and Buildkite/CI wiring using the documentation present on the reviewed
|
||||
branch. A skipped test is not a pass.
|
||||
|
||||
## Documentation sync
|
||||
|
||||
Require user-facing updates when the diff changes a model, feature, CLI/API,
|
||||
config key, default, compatibility behavior, or supported platform. Verify:
|
||||
|
||||
- the actual docs navigation and support tables used by the branch;
|
||||
- exact identifiers, defaults, constraints, known limitations, and migration;
|
||||
- a minimal runnable example for newly exposed behavior;
|
||||
- examples and docs agree on commands, parameters, output schema, and supported
|
||||
modes;
|
||||
- links and the bounded live contract build or resolve correctly.
|
||||
|
||||
Do not request docs for internal behavior with no user-visible contract.
|
||||
|
||||
## PR evidence
|
||||
|
||||
For contributor-run tests or benchmarks, require enough provenance to evaluate
|
||||
the claim: hardware, relevant software versions, exact commands, inputs/model,
|
||||
result, and approximate runtime. Performance or accuracy evidence must compare
|
||||
base and head under equivalent conditions; use
|
||||
[perf-verification.md](perf-verification.md).
|
||||
|
||||
Turn a missing test or document into a finding only when it protects or explains
|
||||
changed behavior. Name the smallest useful addition rather than requesting a
|
||||
broad coverage expansion.
|
||||
@@ -1,36 +0,0 @@
|
||||
# Active Verification
|
||||
|
||||
Read this reference when a runnable affected path and suitable local or remote
|
||||
hardware are available. Never claim device evidence from static analysis or a
|
||||
simulated backend.
|
||||
|
||||
Official docs: [installation](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/)
|
||||
and [test execution guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_execution_guide/).
|
||||
|
||||
## Choose the narrowest level
|
||||
|
||||
| Available environment | Verification |
|
||||
| --- | --- |
|
||||
| Matching accelerator, model/assets, and server | Targeted unit/integration test plus one representative E2E request. |
|
||||
| Matching accelerator without serving setup | Targeted model/runtime test and output or metric inspection. |
|
||||
| CPU/static environment only | Import/version preflight, focused CPU tests, config/registry resolution, and static contract checks. |
|
||||
| Missing dependencies or inaccessible hardware | Record the exact gap and evaluate contributor/CI evidence. |
|
||||
|
||||
## Execute safely
|
||||
|
||||
1. Bind the run to the frozen head SHA and record Python, platform, dependency,
|
||||
model, device, and relevant environment fingerprints.
|
||||
2. Run the smallest affected unit test before an E2E workload.
|
||||
3. Exercise the changed public or production-dispatch path with bounded inputs.
|
||||
4. Compare actual output, metrics, cleanup, and exit behavior with the PR claim.
|
||||
5. For base/head claims, use the equivalent A/B protocol in
|
||||
[perf-verification.md](perf-verification.md).
|
||||
|
||||
Avoid disturbing shared servers, ports, models, or accelerators without user
|
||||
authorization. Do not install large dependencies, download gated assets, or
|
||||
start remote workloads merely because hardware might exist.
|
||||
|
||||
Report the command, environment, result, and any skipped path. Classify failures
|
||||
as change-induced, pre-existing, infrastructure, environment, or flaky before
|
||||
making a finding. A hardware gap lowers confidence; it is not automatic proof
|
||||
of a defect.
|
||||
@@ -1,105 +0,0 @@
|
||||
# Maintainer Review Style
|
||||
|
||||
Use this reference only when turning proved findings into comments. The baseline
|
||||
maintainer samples favor short, direct comments and reserve long explanations
|
||||
for architecture or non-obvious failure paths.
|
||||
|
||||
Official docs: [contributing and review process](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/).
|
||||
|
||||
## Evidence basis
|
||||
|
||||
This calibration used 928 PRs merged from 2026-05-10 through 2026-08-08.
|
||||
Candidates were ranked by substantive human inline-review threads, then by
|
||||
distinct reviewers and review rounds. Author self-comments, bots and AI
|
||||
reviewers, and bodyless approvals were excluded. Counts selected the sample;
|
||||
the practices below came from reading root comments and follow-up replies.
|
||||
|
||||
The main set contains the six leaders after filtering; three other high-ranked
|
||||
PRs add core-runtime and benchmark coverage. All were merged.
|
||||
|
||||
| PR | Review boundary |
|
||||
| --- | --- |
|
||||
| [#2906](https://github.com/vllm-project/vllm-omni/pull/2906) | New TTS model, shared utilities, recipes, and request state. |
|
||||
| [#3907](https://github.com/vllm-project/vllm-omni/pull/3907) | Full-duplex API, session lifecycle, concurrency, and realtime evidence. |
|
||||
| [#2162](https://github.com/vllm-project/vllm-omni/pull/2162) | Diffusion world model, OpenPI serving, tests, examples, and docs. |
|
||||
| [#3454](https://github.com/vllm-project/vllm-omni/pull/3454) | Diffusion model, generic CLI surface, loaders, and configuration scope. |
|
||||
| [#4425](https://github.com/vllm-project/vllm-omni/pull/4425) | Structured configuration, typing, defaults, and compatibility. |
|
||||
| [#5397](https://github.com/vllm-project/vllm-omni/pull/5397) | Distributed offload, collectives, failure handling, and load semantics. |
|
||||
| [#4281](https://github.com/vllm-project/vllm-omni/pull/4281) | Composable parallelism, configuration contracts, and examples. |
|
||||
| [#3628](https://github.com/vllm-project/vllm-omni/pull/3628) | Stage metrics, benchmark definitions, opt-in API behavior, and cleanup. |
|
||||
| [#3855](https://github.com/vllm-project/vllm-omni/pull/3855) | Stage runtime, thread safety, startup, and compatibility. |
|
||||
|
||||
Historical comments calibrate delivery; they do not create merge policy. Live
|
||||
code, tests, repository docs, and the current `CODEOWNERS` file remain
|
||||
authoritative. When refreshing this study, promote a practice only when it
|
||||
recurs across at least three PRs or is supported by current repository guidance.
|
||||
|
||||
## Recurring review behavior
|
||||
|
||||
- Start from a changed contract, not a style preference. State the exact
|
||||
request, configuration, topology, or failure path that triggers the issue.
|
||||
- Protect repository boundaries. Keep model-specific behavior in its domain,
|
||||
move genuinely reusable logic to a shared module, and reuse existing loaders,
|
||||
protocols, definitions, and config sources instead of bypassing them.
|
||||
- Challenge unrelated scope and abstractions. Ask why a new flag, config, class,
|
||||
helper, fallback, or file is needed when an existing path appears sufficient.
|
||||
- Trace concurrency and terminal paths: overlapping requests, disconnect,
|
||||
cancellation, partial failure, stale replies, collective schedule mismatch,
|
||||
cleanup, and retry. Prefer an explicit startup error for unsupported
|
||||
combinations over silently running with incomplete state or wrong weights.
|
||||
- Preserve user-visible behavior across variants. An accepted CLI/API/config
|
||||
field must be applied, explicitly rejected, or visibly warned about; compare
|
||||
normal and headless, offline and online, and streaming and non-streaming paths.
|
||||
- Ask for evidence at the affected boundary: a regression test using the real
|
||||
tensor or protocol shape, correct pytest markers so CI runs it, comparable A/B
|
||||
performance results, or an SLO-oriented latency/quality measurement.
|
||||
- Keep interfaces explicit: avoid `Any` and opaque strings where a precise
|
||||
union, enum, or typed envelope exists; remove dead code and duplicated
|
||||
constants; validate requests before expensive execution.
|
||||
- Document module scope and non-obvious invariants. Update examples, recipes,
|
||||
supported-model/feature docs, and hardware assumptions when users rely on
|
||||
them.
|
||||
- When expertise crosses an ownership boundary, ask a named owner to inspect a
|
||||
specific file or contract and explain why. Do not send a generic `PTAL` list.
|
||||
- Keep non-blocking cleanup in a linked follow-up only after the current PR is
|
||||
safe. If a revision fixes only part of a failure chain, continue the existing
|
||||
thread with the remaining trigger instead of opening a duplicate finding.
|
||||
|
||||
## Write the finding
|
||||
|
||||
- Lead with the issue or decisive question; omit a review preamble.
|
||||
- Name the concrete trigger and impact, then the smallest fix direction.
|
||||
- Keep obvious fixes to one sentence. Use more detail only when the call path or
|
||||
ownership boundary is otherwise unclear.
|
||||
- Prefer one root-cause comment over repeated symptom comments.
|
||||
- Hedge only when evidence is genuinely incomplete; mark that item as a
|
||||
question or validation gap rather than overstating severity.
|
||||
- Say briefly that no findings were found when the review is clean; zero
|
||||
comments is a valid outcome when posting is authorized.
|
||||
|
||||
Useful forms:
|
||||
|
||||
```text
|
||||
This leaves <state> allocated when <failure/cancellation path>. Release it in
|
||||
the shared terminal cleanup path.
|
||||
```
|
||||
|
||||
```text
|
||||
Does this producer also update <consumer>? It still reads the old field here.
|
||||
```
|
||||
|
||||
```text
|
||||
With <configuration>, this bypasses <existing loader/validator>, so <impact>.
|
||||
Reuse that path or reject this combination before startup.
|
||||
```
|
||||
|
||||
```text
|
||||
@<owner>, could you check <specific boundary>? This changes <contract> used by
|
||||
<consumer>.
|
||||
```
|
||||
|
||||
Use a priority label only for a proved high-impact defect. Avoid generic praise,
|
||||
routine “Nit:” prefixes, decorative severity labels, audit templates, rule IDs,
|
||||
comment-count narration, and “I left a few comments.” Keep approvals, review
|
||||
events, reviewer requests, mentions, and GitHub replies subject to the user's
|
||||
explicit authorization.
|
||||
@@ -1,99 +0,0 @@
|
||||
# Focused Code-Owner Review Requests
|
||||
|
||||
Read only when the user asks to identify, suggest, request, add, or ping
|
||||
reviewers. Finding candidates is read-only; GitHub review requests and
|
||||
`@mention` comments are external writes.
|
||||
|
||||
Owner sources, in order:
|
||||
|
||||
1. the reviewed head's
|
||||
[CODEOWNERS](https://github.com/vllm-project/vllm-omni/blob/main/.github/CODEOWNERS)
|
||||
for path ownership;
|
||||
2. the active module design page's `owners` and `required_reviewers` metadata
|
||||
for contract ownership and mandatory design review; and
|
||||
3. documented
|
||||
[governance expertise](https://docs.vllm.ai/projects/vllm-omni/en/latest/community/governance/)
|
||||
for specialization and tie-breaking.
|
||||
|
||||
Resolve all three from the reviewed head when available because ownership
|
||||
changes over time. Use published docs only to discover a page absent from an
|
||||
older checkout, never to override the frozen head, and do not copy a static
|
||||
owner list into review output. CODEOWNERS remains the path authority; module
|
||||
metadata identifies contract experts and required design reviewers.
|
||||
|
||||
## Rank candidates
|
||||
|
||||
1. Freeze the PR head and collect changed files, author, requested reviewers,
|
||||
submitted reviews, and existing comments.
|
||||
2. Resolve the **last matching CODEOWNERS rule** for each changed file. Do not
|
||||
union an earlier broad rule with a later specific rule.
|
||||
3. Resolve the primary module and feature overlays through
|
||||
[design-contracts.md](../process/design-contracts.md). Read the active module
|
||||
page's status, `owners`, and `required_reviewers`; do not treat owners on an
|
||||
archived, superseded, or unrelated page as current authority.
|
||||
4. Group files by the selected primary owner and the exact module or feature
|
||||
contract needing review. Weight production owners and live consumers above
|
||||
tests/docs/CI owners and the repository-wide fallback. Include an applicable
|
||||
`required_reviewers` entry even when it is not the path owner, and label why.
|
||||
5. Use documented governance specialties only to choose among matching owners:
|
||||
for example serving versus configuration, diffusion model versus cache or
|
||||
parallelism, TTS versus generic model execution, or CI versus tests.
|
||||
6. Exclude the PR author, bots, users who already completed the needed review,
|
||||
and duplicate pending requests. Do not infer availability or invent
|
||||
expertise from a username.
|
||||
7. Select one reviewer for a single-owner change and at most three for a real
|
||||
cross-module change. If more are plausible, rank them but do not ping a
|
||||
broad group; ask the user before exceeding three.
|
||||
|
||||
If every specific owner is excluded, consider one repository-wide fallback
|
||||
owner and label that choice as a fallback rather than a path match. Never fan
|
||||
out to the full fallback list. Prefer a qualified owner who covers multiple
|
||||
critical path groups over separate reviewers for each file. If CODEOWNERS is
|
||||
missing or invalid, state that gap and use governance only as a labeled fallback.
|
||||
|
||||
Recent file history may break a tie only when CODEOWNERS and governance remain
|
||||
ambiguous. Never use commit count alone to override an explicit current owner.
|
||||
|
||||
## Preview before writing
|
||||
|
||||
Show the user the exact candidate set and rationale:
|
||||
|
||||
```text
|
||||
@owner — <owned paths or module role>; please verify <specific module or feature contract>.
|
||||
```
|
||||
|
||||
Draft one consolidated comment tied to the frozen head:
|
||||
|
||||
```text
|
||||
Requesting focused review for `<short-head-sha>`:
|
||||
|
||||
- @owner1: could you check the <module contract> in `<path/group>`?
|
||||
- @owner2: could you check the <feature or cross-boundary contract> in `<path/group>`?
|
||||
|
||||
Relevant validation: <one concise result or exact remaining gap>.
|
||||
```
|
||||
|
||||
Mention only owners being requested. Keep the reason specific enough that each
|
||||
person knows what decision or invariant needs attention; do not post a generic
|
||||
“please review” or a generated ownership audit.
|
||||
|
||||
## External-write gate
|
||||
|
||||
- “Who should review?”, “suggest reviewers”, or “draft a request” authorizes
|
||||
only the preview.
|
||||
- “Request/add these reviewers” authorizes the GitHub review-request mutation.
|
||||
- “Ping/comment/tag the owners” authorizes the consolidated comment.
|
||||
- When the user explicitly asks to request reviews **with an owner comment**,
|
||||
both writes are authorized. Obey narrower wording when only one action is
|
||||
requested.
|
||||
|
||||
Immediately before an authorized write, verify the PR is still open, the head
|
||||
SHA is unchanged, and the candidate is not the author or already satisfied.
|
||||
Report draft/WIP state and avoid notifications unless the user explicitly wants
|
||||
reviewers requested before ready-for-review.
|
||||
|
||||
Use the GitHub connector for reviewer requests and the single top-level comment;
|
||||
use `gh` only when connector coverage is unavailable. Do not submit an approval,
|
||||
request-changes event, label, assignment, or multiple comments as part of this
|
||||
action. After writing, reread the PR and report the reviewers requested and the
|
||||
comment URL. If the head changed, discard the draft and rerank before posting.
|
||||
@@ -1,24 +0,0 @@
|
||||
# Communication Features
|
||||
|
||||
Use with the OmniConnector, stage-runtime, configuration, and I/O module
|
||||
contracts. Load the concrete backend design from the reviewed head:
|
||||
|
||||
| Backend | Design |
|
||||
| --- | --- |
|
||||
| Mooncake store | [MooncakeStoreConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mooncake_store_connector/) |
|
||||
| Mooncake transfer engine | [MooncakeTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mooncake_transfer_engine_connector/) |
|
||||
| Mori transfer engine | [MoriTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mori_transfer_engine_connector/) |
|
||||
| Shared memory | [SharedMemoryConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/shared_memory_connector/) |
|
||||
| Yuanrong store | [YuanrongConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/yuanrong_connector/) |
|
||||
| Yuanrong transfer engine | [YuanrongTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/yuanrong_transfer_engine_connector/) |
|
||||
|
||||
## Feature checks
|
||||
|
||||
- Verify dependency, service, device/network, port, memory-pool, and environment
|
||||
prerequisites with actionable startup errors.
|
||||
- Match key/metadata construction, sender and receiver roles, allocation,
|
||||
ownership, completion, and supported fast/serialized paths.
|
||||
- Exercise timeout, retry, stale sender/buffer reclamation, partial transfer,
|
||||
cancellation, and connector close semantics.
|
||||
- Validate the smallest supported same-host or cross-node topology and document
|
||||
backend limitations and operational diagnostics.
|
||||
@@ -1,26 +0,0 @@
|
||||
# Diffusion Acceleration Features
|
||||
|
||||
Use with the exact diffusion submodule plus platform, cache, quantization, or
|
||||
benchmarking contracts selected by the live path.
|
||||
|
||||
| Feature family | Designs |
|
||||
| --- | --- |
|
||||
| Parallelism | [CFG](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cfg_parallel/), [Expert](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/expert_parallel/), [HSDP](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/hsdp/), [Pipeline](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/pipeline_parallel/), [Sequence](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/sequence_parallel/), [Tensor](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/tensor_parallel/), [VAE patch](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/vae_parallel/) |
|
||||
| Attention | [Skip-Softmax](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/skip_softmax/) |
|
||||
| Quantization | [Quantization](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/quantization/) |
|
||||
| Cache | [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cache_dit/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/teacache/) |
|
||||
| Batching | [Diffusion Continuous Batching](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/diffusion_continuous_batching/) |
|
||||
| Offload | [Distributed Layerwise Offload](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/distributed_layerwise_offload/) |
|
||||
|
||||
## Feature checks
|
||||
|
||||
- Verify feature selection, supported models/platforms/topologies, defaults,
|
||||
dependencies, and every touched feature combination.
|
||||
- Preserve feature-off and single-rank correctness; reject unsupported
|
||||
combinations before collectives, loading, or execution begin.
|
||||
- Trace shapes, shards, ranks, collectives, precision, cache validity, residency,
|
||||
and per-request state through the selected runtime hooks.
|
||||
- Compare output quality or accuracy against the disabled/reference path before
|
||||
accepting latency or memory improvements.
|
||||
- Require exact commands, topology, environment, warmup, repetitions, and
|
||||
measurements for performance claims; update feature compatibility docs.
|
||||
@@ -1,22 +0,0 @@
|
||||
# Infrastructure and Performance Features
|
||||
|
||||
Load the matching design from the reviewed head:
|
||||
|
||||
- [Prometheus Metrics](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/metrics/)
|
||||
for logger wrapping, gating, lifecycle, stage transfer, and metric definitions.
|
||||
- [Speech Generation Performance Optimizations](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/qwen3_omni_tts_performance_optimization/)
|
||||
for stage batching, CUDA graphs, async chunk, streaming, re-prefill, compile,
|
||||
and deployment evidence.
|
||||
|
||||
## Feature checks
|
||||
|
||||
- Route instrumentation changes through observability or profiling and result
|
||||
methodology through benchmarking and the performance evidence check.
|
||||
- Preserve opt-in/default behavior, units, labels, aggregation, timing scope,
|
||||
lifecycle resets, and bounded overhead.
|
||||
- For stacked optimizations, measure each increment against the same baseline
|
||||
and explain interactions, quality guards, regressions, and limitations.
|
||||
- Require exact model, checkpoint, hardware, software, workload, topology,
|
||||
concurrency, warmup, repetitions, and production commands.
|
||||
- Update the design page when metric meaning, supported optimization stack, or
|
||||
deployment guidance changes.
|
||||
@@ -1,24 +0,0 @@
|
||||
# Runtime and Stage Execution Features
|
||||
|
||||
Load the matching design from the reviewed head:
|
||||
|
||||
| Feature | Design | Primary module intersections |
|
||||
| --- | --- | --- |
|
||||
| Disaggregated inference | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/disaggregated_inference/) | Configuration, stage runtime, OmniConnector, I/O contracts |
|
||||
| Async chunk | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/async_chunk/) | Model integration, orchestration, stage runtime, connector, I/O |
|
||||
| Async diffusion output | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/async_diffusion_output/) | Diffusion runtime, I/O contracts, entrypoints |
|
||||
| Async Omni output materialization | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_async_output_materialization/) | AR runtime, model integration, I/O contracts, orchestration |
|
||||
| Automatic prefix caching | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/prefix_caching/) | AR runtime, cache management, I/O contracts |
|
||||
|
||||
## Feature checks
|
||||
|
||||
- Confirm the feature's supported models, entrypoints, stage types, topology,
|
||||
configuration, default, and compatibility limitations.
|
||||
- Trace enablement from public/config ingress through every participating module
|
||||
and verify the advertised production dispatch actually selects it.
|
||||
- Preserve a correct disabled or fallback path and make unsupported combinations
|
||||
fail or warn as documented.
|
||||
- Test overlap, ordering, identity, backpressure, cancellation, failure, final
|
||||
partial output, and cleanup at each asynchronous boundary.
|
||||
- Update the feature page, support matrices, examples, and recipes when behavior
|
||||
or coverage changes.
|
||||
@@ -1,22 +0,0 @@
|
||||
# Autoregressive Runtime
|
||||
|
||||
Primary design: [Autoregressive Runtime](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/ar_runtime/).
|
||||
|
||||
Use for AR request queues, scheduling state, token and memory budgets, KV/input
|
||||
readiness, workers, runners, and upstream vLLM request/cache transitions.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Preserve upstream request-state, scheduling, cache, and preemption semantics
|
||||
unless an Omni-specific difference is documented and tested.
|
||||
- Convert modality-specific data through explicit input or output adapters; do
|
||||
not inject it through unrelated scheduler state.
|
||||
- Keep workers and model runners focused on assigned execution rather than
|
||||
admission or cross-stage routing.
|
||||
- Synchronize token, sequence, batch, and memory budgets with worker-visible
|
||||
state and prevent starvation, double counting, or stale readiness.
|
||||
- Preserve legal transitions and one terminal cleanup path across completion,
|
||||
abort, timeout, preemption, worker failure, and shutdown.
|
||||
|
||||
Test request lifecycle, scheduling decisions, abort, cache state, adapters, and
|
||||
every affected worker execution mode against the pinned upstream contract.
|
||||
@@ -1,24 +0,0 @@
|
||||
# Benchmarking
|
||||
|
||||
Primary design: [Benchmarking](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/benchmarking/).
|
||||
|
||||
Use when benchmark code, workloads, metric calculations, CLI benchmark
|
||||
entrypoints, or result metadata change. Use the performance check separately
|
||||
when a PR uses benchmark results as evidence.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Record model and revision, code SHA, hardware, software, configuration,
|
||||
workload, precision, topology, concurrency, warmup, repetitions, and
|
||||
measurement window.
|
||||
- Validate successful and semantically acceptable outputs before accepting
|
||||
timing, memory, or scaling results.
|
||||
- Define every metric's unit, population, aggregation, synchronization, and
|
||||
timing scope.
|
||||
- Keep workload parsing and metric computation deterministic and reusable
|
||||
across base and head.
|
||||
- Reject comparisons that change material workload or runtime settings without
|
||||
isolating and explaining that difference.
|
||||
|
||||
Test workload parsing and metric calculations with deterministic fixtures and
|
||||
publish exact commands plus variability for quantitative claims.
|
||||
@@ -1,22 +0,0 @@
|
||||
# Cache Management
|
||||
|
||||
Primary design: [Cache Management](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/cache_management/).
|
||||
|
||||
Use for AR or diffusion cache identity, reusable state, validity, lifecycle,
|
||||
reset, eviction, and teardown.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Include every model, request, input, timestep, precision, device, topology,
|
||||
and feature property that can change a reused value in cache identity.
|
||||
- Preserve a correct disabled and miss path; never reuse after validity inputs
|
||||
change.
|
||||
- Isolate request-scoped state across concurrent requests and define ownership
|
||||
for model-scoped state.
|
||||
- Make reset, invalidation, eviction, capacity, and teardown explicit on
|
||||
completion, cancellation, failure, model reload, and shutdown.
|
||||
- Keep cache optimizations behind the owning runtime's hooks rather than adding
|
||||
a second request lifecycle.
|
||||
|
||||
Test hit, miss, invalidation, disabled behavior, concurrency, capacity, reset,
|
||||
and teardown, with quality comparison when reuse is approximate.
|
||||
@@ -1,24 +0,0 @@
|
||||
# vLLM-Omni Configuration
|
||||
|
||||
Primary design: [vLLM-Omni Configuration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/vllm_omni_config/).
|
||||
|
||||
Use for config construction, deploy YAML, registries, schema, defaults,
|
||||
aliases, CLI projection, pipeline config, and topology. Inspect the reviewed
|
||||
page's status: while its contract remains deferred or draft, current code and
|
||||
tests—not proposed precedence rules—are authoritative.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Resolve each supported structured, legacy, CLI, environment, default, alias,
|
||||
and stage-override producer through its actual runtime consumer.
|
||||
- Reject unknown or owner-mismatched fields explicitly; keep process-local
|
||||
runtime objects out of transportable configuration.
|
||||
- Verify default, explicit, and feature-off values reach every live factory and
|
||||
consumer without silent reinterpretation.
|
||||
- Treat stage count, placement, connector, parallelism, device, and memory
|
||||
fields as one topology contract and reject impossible combinations early.
|
||||
- Preserve supported construction-path parity until a documented migration
|
||||
removes a path.
|
||||
|
||||
Require normalization and schema tests, a live-consumer assertion, and docs for
|
||||
public keys, defaults, constraints, precedence, or migration behavior.
|
||||
@@ -1,33 +0,0 @@
|
||||
# Diffusion Module Family
|
||||
|
||||
Primary design: [Diffusion Modules](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/).
|
||||
|
||||
Use for diffusion-stage behavior, then load the exact submodule page from the
|
||||
reviewed head:
|
||||
|
||||
| Submodule | Signals | Design |
|
||||
| --- | --- | --- |
|
||||
| Runtime | Admission, scheduling, execution, progress, outputs, cancellation, hooks, cleanup | [Diffusion Runtime](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/diffusion_runtime/) |
|
||||
| Model Integration | Pipelines, registry, loader, adapters, shared layers, model-specific processing | [Diffusion Model Integration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/diffusion_model_integration/) |
|
||||
| Continuous Batching | Compatibility, request/step batches, per-request progress, capacity | [Continuous Batching](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/continuous_batching/) |
|
||||
| Parallelism | Rank topology, process groups, sharding, collectives, distributed execution | [Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/parallelism/) |
|
||||
| Offloader | Residency, transfers, memory accounting, prefetch, teardown | [Offloader](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/offloader/) |
|
||||
|
||||
## Family-wide checks
|
||||
|
||||
- Keep one scheduler-owned lifecycle from admission through completion,
|
||||
cancellation, failure, and cleanup; optional features must use defined hooks.
|
||||
- Select pipelines through registry and loader contracts instead of scattered
|
||||
model-name conditionals; keep genuine model differences in model directories.
|
||||
- Trace latent, conditioning, timestep, generator, shape, layout, dtype, device,
|
||||
batch expansion, and output conversion through the actual consumer.
|
||||
- Make batch compatibility and per-request isolation explicit; never associate
|
||||
output by unstable batch position.
|
||||
- Derive topology from validated configuration, define every sharding boundary,
|
||||
keep collectives symmetric, and preserve a supported single-rank path.
|
||||
- Track one residency owner per offloaded component, wait for transfer readiness,
|
||||
preserve model state, bound retained copies, and clean up deterministically.
|
||||
|
||||
Load matching feature designs for cache, quantization, parallelism, continuous
|
||||
batching, or offload. Require feature-off correctness, representative inference,
|
||||
terminal-path tests, and quality evidence for optimizations.
|
||||
@@ -1,23 +0,0 @@
|
||||
# Engine Orchestration
|
||||
|
||||
Primary design: [Engine Orchestration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/engine_orchestration/).
|
||||
|
||||
Use for request-state creation, cross-stage routing, output ordering, companion
|
||||
tracking, RPC correlation, cancellation propagation, and terminal convergence.
|
||||
Orchestration does not own replica placement, stage-process startup, public
|
||||
rendering, connector transport, or payload and error schemas.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Keep one owner for cross-stage routing; entrypoints, models, and stage clients
|
||||
must not independently select the next logical stage.
|
||||
- Preserve request identity and output order across concurrent stages and
|
||||
correlate every control or RPC result to the originating request and wave.
|
||||
- Make terminal state monotonic: no new forwarding after success, abort,
|
||||
request failure, or fatal engine failure.
|
||||
- Propagate cancellation and failure to every active stage and release
|
||||
orchestrator-owned tasks, queues, companions, and request state.
|
||||
- Preserve shutdown ordering across the orchestrator and stage runtime.
|
||||
|
||||
Test representative multi-stage routing, ordering, correlation, cancellation,
|
||||
failure, and repeated shutdown.
|
||||
@@ -1,25 +0,0 @@
|
||||
# Entrypoints and Serving Boundaries
|
||||
|
||||
Primary design: [Entrypoints and Serving Boundaries](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/entrypoints/).
|
||||
|
||||
Use for offline APIs, CLI and server composition, OpenAI-compatible routes,
|
||||
request validation, response conversion, streaming, sessions, and engine
|
||||
handoff. Entrypoints adapt public contracts; they do not own cross-stage
|
||||
routing, stage lifecycle, configuration precedence, payload schemas, or error
|
||||
classification.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Normalize and validate public values once before engine submission.
|
||||
- Preserve request identity, modality, ordering, and exactly one terminal event
|
||||
across streaming, disconnect, cancellation, and failure.
|
||||
- Translate internal outputs and errors explicitly for each advertised offline,
|
||||
HTTP, SSE, WebSocket, or asynchronous-job path.
|
||||
- Keep model-specific behavior behind a common adapter or processor instead of
|
||||
adding model-name policy to generic entrypoints.
|
||||
- Require API or CLI changes to explain necessity, alternatives, compatibility,
|
||||
migration, and the supported route or transport matrix.
|
||||
|
||||
Test the affected public protocol, normalization, engine handoff, rendering,
|
||||
and terminal outcomes. Route orchestration, I/O schema, and semantic error
|
||||
changes to their module contracts.
|
||||
@@ -1,24 +0,0 @@
|
||||
# Error Classification, Propagation, and Rendering
|
||||
|
||||
Primary design: [Error Classification, Propagation, and Rendering](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/error_contracts/).
|
||||
|
||||
Use when a change classifies failures, crosses process boundaries, decides
|
||||
fatal versus request-scoped impact, sanitizes context, or renders offline,
|
||||
HTTP, SSE, WebSocket, or asynchronous-job errors. The I/O module owns the wire
|
||||
schema; this module owns the meaning of its error fields.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Keep request-scoped failure isolated unless the underlying engine is no
|
||||
longer usable.
|
||||
- Preserve stable classification and request context across process and
|
||||
protocol boundaries without exposing prompts, outputs, secrets, or internal
|
||||
stack details.
|
||||
- Keep cancellation distinguishable from internal failure.
|
||||
- Terminate each public transport exactly once with consistent status, body,
|
||||
event, finish reason, and metrics classification.
|
||||
- Maintain compatibility across mixed or older producers only when the
|
||||
reviewed contract defines that behavior.
|
||||
|
||||
Test table-driven classification, wire round trips, fatal/request-scoped
|
||||
propagation, cancellation, sanitization, and every affected public renderer.
|
||||
@@ -1,23 +0,0 @@
|
||||
# Execution Platforms
|
||||
|
||||
Primary design: [Execution Platforms](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/execution_platforms/).
|
||||
|
||||
Use for hardware capability detection, worker selection, patches, kernels,
|
||||
attention backends, and vendor-specific configuration or connectors.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Guard hardware behavior through platform selection or capability detection
|
||||
and fail clearly when unsupported.
|
||||
- Keep portable modules free of direct vendor imports when a common selector
|
||||
exists; verify portable imports without the target accelerator.
|
||||
- Override only behavior that differs from common or upstream implementations,
|
||||
and compare sibling worker hooks when inheritance does not provide parity.
|
||||
- Preserve dtype, layout, rank, stream, synchronization, workspace,
|
||||
graph-capture, and resource-lifetime contracts.
|
||||
- Do not let simplified CPU doubles hide platform MRO, initialization, or
|
||||
dispatch failures.
|
||||
|
||||
Run portable checks first, then the smallest matching-device test in a fresh
|
||||
environment. Report unavailable hardware as a validation gap, never simulated
|
||||
device evidence.
|
||||
@@ -1,23 +0,0 @@
|
||||
# Input, Output, and Modality Contracts
|
||||
|
||||
Primary design: [Input, Output, and Modality Contracts](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/input_output_modality_contracts/).
|
||||
|
||||
Use for requests, prompt and modality types, message schemas, serialization,
|
||||
output types, accumulation, completion, and compatibility shims crossing
|
||||
entrypoint, orchestration, stage, connector, or model boundaries.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Give every boundary value an explicit modality and validated typed schema.
|
||||
- Preserve request and stage identity across conversion, serialization,
|
||||
streaming updates, batching, cancellation, and errors.
|
||||
- Distinguish partial, final, empty, and failed output states for every modality.
|
||||
- Verify producer and consumer agreement on shape, layout, dtype, device,
|
||||
ownership, metadata, ordering, and optional or unknown fields.
|
||||
- Keep internal objects out of public protocols; translate them at the owning
|
||||
entrypoint.
|
||||
- Preserve wire compatibility deliberately and give deprecated shims a bounded
|
||||
migration path.
|
||||
|
||||
Require construction, round-trip, accumulation, completion, rejection, and
|
||||
compatibility tests at each changed producer-consumer boundary.
|
||||
@@ -1,25 +0,0 @@
|
||||
# Model Integration
|
||||
|
||||
Primary design: [Model Integration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/model_integration/).
|
||||
|
||||
Use for model registration, configuration adapters, preprocessing, checkpoint
|
||||
loading, stage inputs, model-specific hooks, shared layers, runners, and model
|
||||
execution. Use the diffusion family reference for diffusion models.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Declare explicitly how the model class, loader, input processor, stage
|
||||
configuration, and runtime capabilities are selected.
|
||||
- Preserve weight-name mapping, dtype/device placement, optional-dependency
|
||||
errors, and checkpoint-specific loading semantics.
|
||||
- Trace each typed input and output through preprocessing, runner state, model
|
||||
call, and its actual consumer.
|
||||
- Reuse upstream vLLM behavior when sufficient; document and test the precise
|
||||
difference for every override or patch.
|
||||
- Keep model code out of admission, cross-stage routing, public policy, and
|
||||
terminal lifecycle ownership.
|
||||
- Place reusable behavior in shared modules and test it with more than one
|
||||
integration.
|
||||
|
||||
Require focused registry, loader, processor, runner, and representative model
|
||||
execution tests. Apply the model-addition check for new or expanded support.
|
||||
@@ -1,21 +0,0 @@
|
||||
# Observability
|
||||
|
||||
Primary design: [Observability](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/observability/).
|
||||
|
||||
Use for metrics, logs, correlation fields, and cross-stage request visibility.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Preserve stable request and stage identity across entrypoints, orchestration,
|
||||
stages, workers, errors, and terminal cleanup.
|
||||
- Treat metric names, units, labels, aggregation, and lifecycle as public
|
||||
contracts; document any semantic change.
|
||||
- Keep labels bounded: never use request IDs, prompts, outputs, paths, or other
|
||||
unbounded user values.
|
||||
- Register and reset metrics exactly once across repeated startup and shutdown,
|
||||
and keep disabled instrumentation inexpensive.
|
||||
- Align emitted values with their actual timing and lifecycle owner rather than
|
||||
reconstructing them from ambiguous client strings.
|
||||
|
||||
Test registration, label sets, units, resets, error classification, and
|
||||
representative multi-stage emission.
|
||||
@@ -1,24 +0,0 @@
|
||||
# OmniConnector
|
||||
|
||||
Primary design: [OmniConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/omni_connector/).
|
||||
|
||||
Use for connector contracts and model-agnostic data transfer across stages,
|
||||
processes, devices, or nodes. Connector implementations transport and
|
||||
synchronize data; they do not choose the next stage or implement model policy.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Prove producer and consumer agreement on request/stage identity, shape,
|
||||
layout, dtype, placement, ownership, ordering, and completion.
|
||||
- Keep backend selection and metadata/key construction explicit; do not silently
|
||||
reinterpret or drop payloads across SHM, store, network, or vendor paths.
|
||||
- Bound connect, put/get, retry, synchronization, and readiness waits with
|
||||
backpressure, cancellation, and actionable errors.
|
||||
- Prevent route, endpoint, port, key, and buffer collisions across repeated
|
||||
startup, concurrent requests, and replica replacement.
|
||||
- Release connections, handles, buffers, listener work, and stale sender state
|
||||
on success, cancellation, failure, and shutdown.
|
||||
|
||||
Load the matching communication feature design for a concrete backend. Require
|
||||
focused producer-consumer tests and the smallest runnable multi-process or
|
||||
multi-node path.
|
||||
@@ -1,21 +0,0 @@
|
||||
# Profiling
|
||||
|
||||
Primary design: [Profiling](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/profiling/).
|
||||
|
||||
Use for opt-in instrumentation and trace collection across supported profiler
|
||||
backends.
|
||||
|
||||
## Contract checks
|
||||
|
||||
- Keep expensive collection disabled by default and bounded by explicit start,
|
||||
stop, timeout, and cleanup behavior.
|
||||
- Preserve scheduling, generated outputs, resource ownership, and concurrency
|
||||
semantics apart from documented synchronization overhead.
|
||||
- Include stage, rank, worker, operation, and request correlation where useful
|
||||
without embedding user payloads or secrets.
|
||||
- Make repeated start/stop and partial failure safe, including multi-rank trace
|
||||
naming and artifact ownership.
|
||||
- Measure disabled and enabled overhead before making performance claims.
|
||||
|
||||
Test the disabled path, repeated lifecycle, failure cleanup, trace context, and
|
||||
each affected backend.
|
||||
Loaded 100 of 3412 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user