tzhouam
a252918008
- Updated the vLLM wheel to commit 24650715105a26b41c88f4164777b5e1a0f3200b in Dockerfile.ci. - Modified the `calculate_metrics` function to handle `tokenizer` as optional (can be None) in metrics.py. - Adjusted the `benchmark` function to accept `tokenizer` as optional in patch.py. - Enhanced condition checks in OmniARScheduler to ensure valid token generation in omni_ar_scheduler.py. - Updated input processing to include supported tasks in input_processor.py. - Cleaned up unnecessary code in output_processor.py. - Added support for SSL ciphers in the API server configuration in api_server.py. - Enabled speculative decoding in GPU model runners for improved performance in gpu_generation_model_runner.py and gpu_model_runner.py. Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
46 lines
2.1 KiB
Docker
46 lines
2.1 KiB
Docker
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||
ARG VLLM_BASE_TAG=v0.15.0
|
||
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
|
||
ARG APP_DIR=/workspace/vllm-omni
|
||
WORKDIR ${APP_DIR}
|
||
|
||
COPY . .
|
||
|
||
# Install system dependencies
|
||
RUN apt-get update && \
|
||
apt-get install -y ffmpeg sox libsox-fmt-all && \
|
||
apt-get clean && \
|
||
rm -rf /var/lib/apt/lists/*
|
||
|
||
# Install vLLM wheel for the specific commit
|
||
# URL format: https://wheels.vllm.ai/<commit_hash>/
|
||
# Current vLLM wheel commit: 24650715105a26b41c88f4164777b5e1a0f3200b
|
||
# Note: --prerelease=allow is required because nightly/commit wheels use
|
||
# pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default,
|
||
# causing it to fall back to the older stable release from PyPI.
|
||
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/24650715105a26b41c88f4164777b5e1a0f3200b
|
||
|
||
|
||
RUN uv pip uninstall --system nvidia-cublas-cu12
|
||
# Install vllm-omni into the same uv-managed Python environment used by the base image.
|
||
# Use bash -c so that $(python3 -c ...) is expanded inside the container.
|
||
RUN uv pip install --system --no-cache-dir ".[dev]"
|
||
|
||
# pplx_kernels is optional; the base image may already ship a version.
|
||
# Reinstall attempt may fail if no compatible wheel exists for the current
|
||
# PyTorch/CUDA combination – allow the build to continue regardless.
|
||
RUN pip install --upgrade --force-reinstall "pplx_kernels" || true
|
||
|
||
RUN /bin/bash -c 'FLASHINFER_CUDA_TAG="$(python3 -c "import torch; print((torch.version.cuda or \"12.4\").replace(\".\", \"\"))")" && \
|
||
uv pip install --system --upgrade --force-reinstall \
|
||
"flashinfer-python==0.6.3" \
|
||
"flashinfer-cubin==0.6.3" \
|
||
"flashinfer-jit-cache==0.6.3" \
|
||
--extra-index-url "https://flashinfer.ai/whl/cu${FLASHINFER_CUDA_TAG}"'
|
||
# Pin nvidia-cublas-cu12 after vllm-omni install so dependency resolution does not overwrite it
|
||
RUN uv pip install --system --upgrade --force-reinstall "nvidia-cublas-cu12==12.9.1.4"
|
||
RUN uv pip install --system --upgrade --force-reinstall "numpy==2.2.6"
|
||
RUN ln -sf /usr/bin/python3 /usr/bin/python
|
||
|
||
ENTRYPOINT []
|