rebase: initialize vLLM v0.29 alignment

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>

docker/Dockerfile.ci

@@ -1,25 +1,17 @@
-# Target vllm version: v0.28.0 (tag v0.28.0 @ 2cf0a6915ce5).
-# Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase.
+# Target vllm version: releases/v0.29.0 @ f5c3cc240bc1, installed on top of
+# the v0.28.0 base image via the precompiled wheel pinned below. The pin and
+# base intentionally differ until an official v0.29.0 base image is published.
 #
-# Release-boundary form: there is no precompiled-wheel pin here, because the
-# base image already ships exactly the target vLLM. Everything the runtime
-# needs — torch, flashinfer, nvJitLink, numpy — comes from that image as one
-# self-consistent set.
+# Pinned form: VLLM_PRECOMPILED_WHEEL_COMMIT is the rebase baseline — the rebase
+# repo-rebase-v3 advances it on each run. Keep the torch/flashinfer/numpy pins
+# aligned with that commit's requirements/cuda.txt and generated test lock.
+# VLLM_BASE_TAG stays at the newest official vLLM release below the pin; the
+# exact vLLM comes from the wheel, not the image.
 #
-# This block reappears only when the rebase targets a vLLM commit that is NOT a
-# released tag. During the 0.27 rebase, targeting v0.27.0rc1 on top of the
-# v0.26.0 base image required reinstalling vLLM from wheels.vllm.ai and then
-# hand-repairing the fallout: pinning torch ahead of flashinfer (installing it
-# after bound flashinfer's TVM FFI runtime to the old torch and aborted engine
-# core init), registering libnvJitLink.so.13 with ldconfig, and re-pinning numpy
-# under numba's ceiling. All of it was compensation for a base image and a vLLM
-# that disagreed. At a real release boundary they agree, so none of it is
-# needed. The 0.28 rebase repeated the cycle: the v0.28.0rc / final-tag wheels
-# rode on the v0.27.0 base image with the full reinstall block until the
-# official v0.28.0 image shipped, at which point this file dropped back to the
-# form you are reading. If you pin an untagged commit again, restore the
-# reinstall block — agent/tasks/20_update_wheel_url.sh fails the run rather
-# than letting the Dockerfile silently ship the wrong vLLM.
+# This block is required whenever the target has no matching official base
+# image, including release candidates. Drop back to the release-boundary form
+# (no wheel pin, no reinstalls, everything from the base image as one
+# self-consistent set) only after the exact tagged image is published.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
 ARG VLLM_BASE_TAG=v0.28.0
 
@@ -60,6 +52,94 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
 # site-packages and breaks ``import vllm_omni.entrypoints``.
 RUN uv pip install --system -e ".[dev]"
 
+# Install vLLM (CUDA 13.0 build) from precompiled wheel at the selected commit.
+# Must use direct URL because the wheel has a PEP 440 local version identifier
+# (e.g. +gaeee7ef93.cu130) which pip/uv refuse to install from a PEP 503 package index.
+# The cu130 wheel index lives at /<commit>/cu130/vllm/ but the actual wheel files
+# are stored at the top level /<commit>/<wheel>.
+ENV VLLM_PRECOMPILED_WHEEL_COMMIT=f5c3cc240bc1fc0519694fe689edb87dee8dfd92
+# Keep the base image's CUDA toolkit libs on the loader path. This does NOT by
+# itself guarantee libnvJitLink.so.13 resolves — the .so.13 JIT linker is
+# handled separately via ldconfig after the cu130 wheels are installed (see the
+# nvidia-nvjitlink-cu13 step below), which is a no-op when the base image
+# already provides it.
+ENV LD_LIBRARY_PATH=/usr/local/cuda/targets/x86_64-linux/lib:${LD_LIBRARY_PATH}
+RUN VLLM_WHEEL_URL=$(python3 -c "import urllib.request,re; \
+    from urllib.parse import urljoin, quote; \
+    index_url='https://wheels.vllm.ai/${VLLM_PRECOMPILED_WHEEL_COMMIT}/cu130/vllm/'; \
+    html=urllib.request.urlopen(index_url).read().decode(); \
+    m=re.search(r'href=\"([^\"]*x86_64\.whl)\"',html); \
+    assert m, f'No x86_64 vLLM wheel found at {index_url}'; \
+    print(quote(urljoin(index_url,m.group(1)), safe=':/+%'))") && \
+    echo "Installing vLLM from: ${VLLM_WHEEL_URL}" && \
+    uv pip install --system --force-reinstall "${VLLM_WHEEL_URL}"
+
+
+# Pin torch BEFORE flashinfer. Versions mirror vLLM requirements/cuda.txt at
+# VLLM_PRECOMPILED_WHEEL_COMMIT.
+#
+# The ordering is the load-bearing part, not the version. When the base image
+# and the pinned wheel agree on torch this pin is a no-op guard; when they
+# diverge it is what keeps flashinfer from resolving against a torch that the
+# vLLM reinstall then removes. That failure mode is not hypothetical: during the
+# 0.27 rebase (v0.27.0rc1 wheel on a v0.26.0 base, torch 2.11.0 -> 2.13.0),
+# installing flashinfer first bound its compiled artifacts — and its bundled TVM
+# FFI runtime — to the old torch, producing a C++ abort during engine-core init:
+#
+#   terminate called after throwing an instance of 'tvm::ffi::Error'
+#     what(): TypeAttr `__ffi_repr__` is already registered for type index 132
+#
+# plus `from deep_ep_cpp import EventHandle` -> ImportError, which together took
+# out 21 CI jobs on build #2947 that pass on main.
+#
+
+# No --upgrade here: with torch already at its pinned version, --upgrade is what
+# would drag it forward again and reintroduce the mismatch this ordering fixes.
+RUN uv pip install --system \
+        "flashinfer-python==0.6.18" \
+        "flashinfer-cubin==0.6.18" \
+        --extra-index-url https://flashinfer.ai/whl/
+
+RUN uv pip install --system \
+    "flashinfer-jit-cache==0.6.18" \
+    --index-url https://flashinfer.ai/whl/cu130
+
+# Fail the build here rather than at engine-core init if anything above moved
+# torch off its pin. The previous failure mode was a C++ terminate inside a
+# worker subprocess, which surfaced only as "Failed core proc(s): {}" — an
+# empty set, hours into the test run.
+RUN python3 -c "import torch, sys; \
+    want='2.13.0'; got=torch.__version__.split('+')[0]; \
+    sys.exit(0) if got==want else sys.exit(f'torch pin drifted: want {want}, got {torch.__version__}')" && \
+    python3 -c "import deep_ep_cpp" 2>/dev/null \
+      && echo 'deep_ep_cpp imports cleanly' \
+      || echo 'WARNING: deep_ep_cpp not importable (non-fatal; vLLM treats deep_ep as optional)'
+
+# CUDA 13 JIT linker: libnvJitLink.so.13 is provided by the nvidia-nvjitlink
+# package that the cu130 torch/vllm wheels above pull into site-packages. When
+# the base image ships a CUDA-12 toolkit, /usr/local/cuda never resolves the
+# .so.13 and the LD_LIBRARY_PATH above is not enough. Even when it is present,
+# its lib dir is not on the dynamic-loader path, so bitsandbytes / torch.compile
+# worker subprocesses fail to dlopen it. Locate the already-installed lib and
+# register its dir with ldconfig (system-wide, survives subprocess env resets).
+# Only fall back to the pip wheel (correct name: nvidia-nvjitlink-cu13, NOT
+# nvidia-cuda-nvjitlink-cu13, which does not exist) if the torch deps did not
+# bring it in.
+RUN NVJITLINK_LIB="$(find / -name 'libnvJitLink.so.13*' 2>/dev/null | head -1)" && \
+    if [ -z "$NVJITLINK_LIB" ]; then \
+        uv pip install --system "nvidia-nvjitlink-cu13" && \
+        NVJITLINK_LIB="$(find / -name 'libnvJitLink.so.13*' 2>/dev/null | head -1)"; \
+    fi && \
+    test -n "$NVJITLINK_LIB" && \
+    dirname "$NVJITLINK_LIB" > /etc/ld.so.conf.d/nvjitlink-cu13.conf && \
+    ldconfig && \
+    echo "Registered ${NVJITLINK_LIB} with ldconfig"
+
+# Re-pin numpy LAST. The v0.29.0rc1 CUDA requirements use numba 0.65.0 and the
+# generated CUDA test lock resolves numpy 2.2.6. This must be the final
+# dependency operation so the wheel and FlashInfer installs cannot clobber it.
+RUN uv pip install --system "numpy==2.2.6"
+
 # The final stage just copies the working dir on top of the
 # dependencies image. This uses --link so that we can skip
 # the image pull in Buildkite when we have a cache hit.