From 80a0d8d20abccd62e0fb81fb5e4b825aa47d5940 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 10 Sep 2026 11:11:42 +0800 Subject: [PATCH] ci: use vLLM 0.29 release image and align installation docs Drop the old-base wheel reinstall and dependency repair block. Update source installation and configuration references to vLLM 0.29 while retaining explicitly matched 0.28 prebuilt Omni examples until its next release. Validation: official image and CUDA/ROCm release artifacts checked; 6 documentation tests passed; diff whitespace checks passed. Markdown lint reports existing snippet/anchor issues. Container build not run: no builder available. Signed-off-by: tzhouam --- docker/Dockerfile.ci | 113 +----------------- docs/configuration/README.md | 2 +- docs/configuration/environment_variables.md | 2 +- docs/getting_started/installation/README.md | 2 +- docs/getting_started/installation/gpu.md | 2 + .../installation/gpu/cuda.inc.md | 22 ++-- .../installation/gpu/rocm.inc.md | 12 +- docs/getting_started/quickstart.md | 6 +- 8 files changed, 33 insertions(+), 128 deletions(-) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 72bd8f5c6..a8e46580b 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -1,19 +1,10 @@ -# Target vllm version: releases/v0.29.0 @ 98dff2a81d74, installed on top of -# the v0.28.0 base image via the precompiled wheel pinned below. The pin and -# base intentionally differ until an official v0.29.0 base image is published. -# -# Pinned form: VLLM_PRECOMPILED_WHEEL_COMMIT is the rebase baseline — the rebase -# repo-rebase-v3 advances it on each run. Keep the torch/flashinfer/numpy pins -# aligned with that commit's requirements/cuda.txt and generated test lock. -# VLLM_BASE_TAG stays at the newest official vLLM release below the pin; the -# exact vLLM comes from the wheel, not the image. -# -# This block is required whenever the target has no matching official base -# image, including release candidates. Drop back to the release-boundary form -# (no wheel pin, no reinstalls, everything from the base image as one -# self-consistent set) only after the exact tagged image is published. +# Target vLLM version: v0.29.0 (tag @ 98dff2a81d74). +# Use the matching release image and its coherent CUDA/PyTorch/FlashInfer +# runtime. Do not reinstall a commit wheel or repair dependencies on top of +# an older base image at a release boundary. For a future unreleased target, +# restore an explicit wheel pin and validate its complete runtime stack. ARG VLLM_BASE_IMAGE=vllm/vllm-openai -ARG VLLM_BASE_TAG=v0.28.0 +ARG VLLM_BASE_TAG=v0.29.0 # We use registry-based layer caching for the dependencies, # where the tag is determined based on the hash of the contents @@ -52,98 +43,6 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 # site-packages and breaks ``import vllm_omni.entrypoints``. RUN uv pip install --system -e ".[dev]" -# Install vLLM (CUDA 13.0 build) from precompiled wheel at the selected commit. -# Must use direct URL because the wheel has a PEP 440 local version identifier -# (e.g. +gaeee7ef93.cu130) which pip/uv refuse to install from a PEP 503 package index. -# The cu130 wheel index lives at //cu130/vllm/ but the actual wheel files -# are stored at the top level //. -ENV VLLM_PRECOMPILED_WHEEL_COMMIT=98dff2a81d747d1dba01a47f939f48c3526d4206 -# Keep the base image's CUDA toolkit libs on the loader path. This does NOT by -# itself guarantee libnvJitLink.so.13 resolves — the .so.13 JIT linker is -# handled separately via ldconfig after the cu130 wheels are installed (see the -# nvidia-nvjitlink-cu13 step below), which is a no-op when the base image -# already provides it. -ENV LD_LIBRARY_PATH=/usr/local/cuda/targets/x86_64-linux/lib:${LD_LIBRARY_PATH} -RUN VLLM_WHEEL_URL=$(python3 -c "import urllib.request,re; \ - from urllib.parse import urljoin, quote; \ - index_url='https://wheels.vllm.ai/${VLLM_PRECOMPILED_WHEEL_COMMIT}/cu130/vllm/'; \ - html=urllib.request.urlopen(index_url).read().decode(); \ - m=re.search(r'href=\"([^\"]*x86_64\.whl)\"',html); \ - assert m, f'No x86_64 vLLM wheel found at {index_url}'; \ - print(quote(urljoin(index_url,m.group(1)), safe=':/+%'))") && \ - echo "Installing vLLM from: ${VLLM_WHEEL_URL}" && \ - uv pip install --system --force-reinstall "${VLLM_WHEEL_URL}" - - -# Pin torch BEFORE flashinfer. Versions mirror vLLM requirements/cuda.txt at -# VLLM_PRECOMPILED_WHEEL_COMMIT. -# -# The ordering is the load-bearing part, not the version. When the base image -# and the pinned wheel agree on torch this pin is a no-op guard; when they -# diverge it is what keeps flashinfer from resolving against a torch that the -# vLLM reinstall then removes. That failure mode is not hypothetical: during the -# 0.27 rebase (v0.27.0rc1 wheel on a v0.26.0 base, torch 2.11.0 -> 2.13.0), -# installing flashinfer first bound its compiled artifacts — and its bundled TVM -# FFI runtime — to the old torch, producing a C++ abort during engine-core init: -# -# terminate called after throwing an instance of 'tvm::ffi::Error' -# what(): TypeAttr `__ffi_repr__` is already registered for type index 132 -# -# plus `from deep_ep_cpp import EventHandle` -> ImportError, which together took -# out 21 CI jobs on build #2947 that pass on main. -# -# Installing torch first means FlashInfer resolves against the final runtime. -RUN uv pip install --system --force-reinstall \ - "torch==2.13.0" "torchaudio==2.11.0" "torchvision==0.28.0" \ - --torch-backend cu130 - -# No --upgrade here: with torch already at its pinned version, --upgrade is what -# would drag it forward again and reintroduce the mismatch this ordering fixes. -RUN uv pip install --system \ - "flashinfer-python==0.6.18" \ - "flashinfer-cubin==0.6.18" \ - --extra-index-url https://flashinfer.ai/whl/ - -RUN uv pip install --system \ - "flashinfer-jit-cache==0.6.18" \ - --index-url https://flashinfer.ai/whl/cu130 - -# Fail the build here rather than at engine-core init if anything above moved -# torch off its pin. The previous failure mode was a C++ terminate inside a -# worker subprocess, which surfaced only as "Failed core proc(s): {}" — an -# empty set, hours into the test run. -RUN python3 -c "import torch, sys; \ - want='2.13.0'; got=torch.__version__.split('+')[0]; \ - sys.exit(0) if got==want else sys.exit(f'torch pin drifted: want {want}, got {torch.__version__}')" && \ - python3 -c "import deep_ep_cpp" 2>/dev/null \ - && echo 'deep_ep_cpp imports cleanly' \ - || echo 'WARNING: deep_ep_cpp not importable (non-fatal; vLLM treats deep_ep as optional)' - -# CUDA 13 JIT linker: libnvJitLink.so.13 is provided by the nvidia-nvjitlink -# package that the cu130 torch/vllm wheels above pull into site-packages. When -# the base image ships a CUDA-12 toolkit, /usr/local/cuda never resolves the -# .so.13 and the LD_LIBRARY_PATH above is not enough. Even when it is present, -# its lib dir is not on the dynamic-loader path, so bitsandbytes / torch.compile -# worker subprocesses fail to dlopen it. Locate the already-installed lib and -# register its dir with ldconfig (system-wide, survives subprocess env resets). -# Only fall back to the pip wheel (correct name: nvidia-nvjitlink-cu13, NOT -# nvidia-cuda-nvjitlink-cu13, which does not exist) if the torch deps did not -# bring it in. -RUN NVJITLINK_LIB="$(find / -name 'libnvJitLink.so.13*' 2>/dev/null | head -1)" && \ - if [ -z "$NVJITLINK_LIB" ]; then \ - uv pip install --system "nvidia-nvjitlink-cu13" && \ - NVJITLINK_LIB="$(find / -name 'libnvJitLink.so.13*' 2>/dev/null | head -1)"; \ - fi && \ - test -n "$NVJITLINK_LIB" && \ - dirname "$NVJITLINK_LIB" > /etc/ld.so.conf.d/nvjitlink-cu13.conf && \ - ldconfig && \ - echo "Registered ${NVJITLINK_LIB} with ldconfig" - -# Re-pin numpy LAST. The v0.29.0rc1 CUDA requirements use numba 0.65.0 and the -# generated CUDA test lock resolves numpy 2.2.6. This must be the final -# dependency operation so the wheel and FlashInfer installs cannot clobber it. -RUN uv pip install --system "numpy==2.2.6" - # The final stage just copies the working dir on top of the # dependencies image. This uses --link so that we can skip # the image pull in Buildkite when we have a cache hit. diff --git a/docs/configuration/README.md b/docs/configuration/README.md index cdfb5818a..3d065f9a1 100644 --- a/docs/configuration/README.md +++ b/docs/configuration/README.md @@ -2,7 +2,7 @@ This section lists the most common options for running vLLM-Omni. -For options within a vLLM Engine, please refer to the [vLLM 0.28 configuration guide](https://docs.vllm.ai/en/v0.28.0/configuration/index.html). +For options within a vLLM Engine, please refer to the [vLLM 0.29 configuration guide](https://docs.vllm.ai/en/v0.29.0/configuration/index.html). Each model defines fixed topology in a registered `PipelineConfig` and runtime overrides in a deploy YAML. diff --git a/docs/configuration/environment_variables.md b/docs/configuration/environment_variables.md index 8f944f44d..90b9ffae0 100644 --- a/docs/configuration/environment_variables.md +++ b/docs/configuration/environment_variables.md @@ -151,7 +151,7 @@ their keys only. ## Inherited vLLM variables vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to -the [vLLM 0.28 environment-variable reference](https://docs.vllm.ai/en/v0.28.0/configuration/env_vars.html) +the [vLLM 0.29 environment-variable reference](https://docs.vllm.ai/en/v0.29.0/configuration/env_vars.html) for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm, XPU, ModelScope, and FlashInfer workspace settings. diff --git a/docs/getting_started/installation/README.md b/docs/getting_started/installation/README.md index 448ca81f2..9d2d11e94 100644 --- a/docs/getting_started/installation/README.md +++ b/docs/getting_started/installation/README.md @@ -1,7 +1,7 @@ # Installation !!! important - vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.28.x requires vLLM 0.28.x; the stable v0.28.0 instructions pin vLLM 0.28.0. + vLLM-Omni is released against the matching upstream vLLM major/minor version. The 0.29 development line requires vLLM 0.29.x; the source-install instructions pin vLLM 0.29.0. Published vLLM-Omni 0.28.0 wheels and images still require vLLM 0.28.x. vLLM-Omni supports the following hardware platforms: diff --git a/docs/getting_started/installation/gpu.md b/docs/getting_started/installation/gpu.md index 7b8778296..e714a52bf 100644 --- a/docs/getting_started/installation/gpu.md +++ b/docs/getting_started/installation/gpu.md @@ -34,6 +34,8 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr ### Pre-built wheels +The source-install instructions target vLLM 0.29.0. Until vLLM-Omni 0.29.0 artifacts are published, the CUDA/ROCm pre-built examples below remain on the matching 0.28.0 release pair. + Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, and 0.28.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source). === "NVIDIA CUDA" diff --git a/docs/getting_started/installation/gpu/cuda.inc.md b/docs/getting_started/installation/gpu/cuda.inc.md index 5e6cd47c9..f6a1004a6 100644 --- a/docs/getting_started/installation/gpu/cuda.inc.md +++ b/docs/getting_started/installation/gpu/cuda.inc.md @@ -5,7 +5,7 @@ # --8<-- [end:requirements] # --8<-- [start:set-up-using-python] -vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.28.x release line uses vLLM 0.28.x. +vLLM-Omni depends on the matching major/minor release of vLLM. The 0.29 development line uses vLLM 0.29.x. Published 0.28.0 wheels and images use vLLM 0.28.x. !!! note PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See for more details. @@ -18,6 +18,8 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new** #### Installation of vLLM +These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.29 development line, use the source-install instructions below. + vLLM-Omni is built based on vLLM. Please install it with command below. ```bash uv pip install vllm==0.28.0 --torch-backend=auto @@ -26,12 +28,12 @@ uv pip install vllm==0.28.0 --torch-backend=auto #### Installation of vLLM-Omni ```bash -uv pip install vllm-omni +uv pip install vllm-omni==0.28.0 ``` To run Gradio demos, also install the optional extras: ```bash -uv pip install 'vllm-omni[demo]' +uv pip install 'vllm-omni[demo]==0.28.0' ``` # --8<-- [end:pre-built-wheels] @@ -39,13 +41,13 @@ uv pip install 'vllm-omni[demo]' # --8<-- [start:build-wheel-from-source] #### Installation of vLLM -If you do not need to modify source code of vLLM, you can directly install the stable 0.28.0 release version of the library +If you do not need to modify source code of vLLM, you can directly install the stable 0.29.0 release version of the library ```bash -uv pip install vllm==0.28.0 --torch-backend=auto +uv pip install vllm==0.29.0 --torch-backend=auto ``` -The 0.28.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead. +The 0.29.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead. #### Installation of vLLM-Omni Since vllm-omni is rapidly evolving, it's recommended to install it from source @@ -66,12 +68,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr ```bash git clone https://github.com/vllm-project/vllm.git cd vllm -git checkout v0.28.0 +git checkout v0.29.0 ``` Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file. ```bash -#For CUDA 13.0 (the default for v0.28.0; the wheel filename has no `+cu130` suffix) -export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.28.0/vllm-0.28.0-cp38-abi3-manylinux_2_28_x86_64.whl +#For CUDA 13.0 (the default for v0.29.0; the wheel filename has no `+cu130` suffix) +export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.29.0/vllm-0.29.0-cp38-abi3-manylinux_2_28_x86_64.whl ``` Install vllm with command below (If you have no existing PyTorch). ```bash @@ -131,7 +133,7 @@ If you want to specify the base vLLM version: ```bash DOCKER_BUILDKIT=1 docker build \ -f docker/Dockerfile.cuda \ - --build-arg BASE_IMAGE=vllm/vllm-openai:v0.28.0 \ + --build-arg BASE_IMAGE=vllm/vllm-openai:v0.29.0 \ -t vllm-omni-cuda . ``` diff --git a/docs/getting_started/installation/gpu/rocm.inc.md b/docs/getting_started/installation/gpu/rocm.inc.md index e0f1614ae..d15bba2eb 100644 --- a/docs/getting_started/installation/gpu/rocm.inc.md +++ b/docs/getting_started/installation/gpu/rocm.inc.md @@ -11,6 +11,8 @@ vLLM-Omni current recommends the steps in under setup through Docker Images. #### Installation of vLLM +These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.29 development line, use the source-install instructions below. + vLLM-Omni is built based on vLLM. Please install it with command below. ```bash uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723 @@ -22,7 +24,7 @@ uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/roc # we need to add --no-build-isolation as the torch # is not obtained from pypi, we have to install using the # torch installed in our environment -uv pip install vllm-omni +uv pip install vllm-omni==0.28.0 # Optional if want to run Qwen3 TTS uv pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm @@ -34,13 +36,13 @@ uv pip install onnxruntime-rocm # --8<-- [start:build-wheel-from-source] #### Installation of vLLM -If you do not need to modify source code of vLLM, you can directly install the stable 0.28.0 release version of the library +If you do not need to modify source code of vLLM, you can directly install the stable 0.29.0 release version of the library ```bash -uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723 +uv pip install vllm==0.29.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.29.0/rocm723 ``` -The pre-built 0.28.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead. +The pre-built 0.29.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead. #### Installation of vLLM-Omni Since vllm-omni is rapidly evolving, it's recommended to install it from source @@ -58,7 +60,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr ```bash git clone https://github.com/vllm-project/vllm.git cd vllm -git checkout v0.28.0 +git checkout v0.29.0 python3 -m pip install -r requirements/rocm.txt python3 setup.py develop ``` diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md index 036c2da17..8363e05ff 100644 --- a/docs/getting_started/quickstart.md +++ b/docs/getting_started/quickstart.md @@ -19,10 +19,10 @@ uv venv --python 3.12 --seed source .venv/bin/activate # On CUDA -uv pip install vllm==0.28.0 --torch-backend=auto +uv pip install vllm==0.29.0 --torch-backend=auto # On ROCm -uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723 +uv pip install vllm==0.29.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.29.0/rocm723 git clone https://github.com/vllm-project/vllm-omni.git cd vllm-omni @@ -34,7 +34,7 @@ For additional installation methods — please see the [installation guide](inst !!! note It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni. - If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.28.0` and vLLM Omni `0.28.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue. + If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.29.0` and vLLM Omni `0.29.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue. ## Offline Inference