diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index a0b4216ff..4c49b9304 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -18,18 +18,18 @@ RUN apt-get update && \ # Note: --prerelease=allow is required because nightly/commit wheels use # pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default, # causing it to fall back to the older stable release from PyPI. -RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/e94ec597334d9a3e9b0d04bc17152e2747c83d51 +RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/4a1550d22d7058e129d0e1257e726b3bf4a77025 # Keep flashinfer runtime packages in version sync. # We derive the version from installed flashinfer-python (pulled by vllm) # and enforce the same version for flashinfer-cubin / flashinfer-jit-cache. -RUN FLASHINFER_VERSION="$(python3 -c 'import importlib.metadata as m; print(m.version("flashinfer-python"))')" && \ - FLASHINFER_CUDA_TAG="$(python3 -c 'import torch; print((torch.version.cuda or "12.4").replace(".", ""))')" && \ - uv pip install --system --upgrade --force-reinstall \ - "flashinfer-cubin==${FLASHINFER_VERSION}" \ - "flashinfer-jit-cache==${FLASHINFER_VERSION}" \ - --extra-index-url "https://flashinfer.ai/whl/cu${FLASHINFER_CUDA_TAG}" && \ - flashinfer show-config +# RUN FLASHINFER_VERSION="$(python3 -c 'import importlib.metadata as m; print(m.version("flashinfer-python"))')" && \ +# FLASHINFER_CUDA_TAG="$(python3 -c 'import torch; print((torch.version.cuda or "12.4").replace(".", ""))')" && \ +# uv pip install --system --upgrade --force-reinstall \ +# "flashinfer-cubin==${FLASHINFER_VERSION}" \ +# "flashinfer-jit-cache==${FLASHINFER_VERSION}" \ +# --extra-index-url "https://flashinfer.ai/whl/cu${FLASHINFER_CUDA_TAG}" && \ +# flashinfer show-config # Install vllm-omni into the same uv-managed Python environment used by the base image. RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]" diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 4e903cafc..760bacf00 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -245,10 +245,10 @@ class OmniARScheduler(VLLMScheduler): # Skip requests that were recovered from KV load failure continue request = self.requests.get(req_id) - if request is None: + if request is None or request.is_finished(): # The request is already finished. This can happen if the # request is aborted while the model is executing it (e.g., - # in pipeline parallelism). + # in pipeline parallelism or async scheduling). continue req_index = model_runner_output.req_id_to_index[req_id] @@ -475,15 +475,16 @@ class OmniARScheduler(VLLMScheduler): return engine_core_outputs - def _free_request(self, request: Request) -> dict[str, Any] | None: + def _free_request(self, request: Request, delay_free_blocks: bool = False) -> dict[str, Any] | None: # TODO(wzliu)! for offline mode, we should not end process until all data is transferred """Mark a request as finished and free its resources.""" + assert request.is_finished() # 1. Standard cleanup parts from base _free_request - delay_free_blocks = False + connector_delay_free_blocks = False kv_xfer_params = None if self.connector is not None: - delay_free_blocks, kv_xfer_params = self._connector_finished(request) + connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request) self.encoder_cache_manager.free(request) request_id = request.request_id @@ -543,6 +544,7 @@ class OmniARScheduler(VLLMScheduler): return kv_xfer_params # 3. Standard Freeing + delay_free_blocks |= connector_delay_free_blocks if not delay_free_blocks: self._free_blocks(request) diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index 61aa70970..4b0b93726 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -317,10 +317,10 @@ class OmniGenerationScheduler(VLLMScheduler): # Skip requests that were recovered from KV load failure continue request = self.requests.get(req_id) - if request is None: + if request is None or request.is_finished(): # The request is already finished. This can happen if the # request is aborted while the model is executing it (e.g., - # in pipeline parallelism). + # in pipeline parallelism or async scheduling). continue req_index = model_runner_output.req_id_to_index[req_id] diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index 9971754e3..f83998baf 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -1,8 +1,10 @@ from dataclasses import dataclass, field from typing import Any +import vllm.envs as envs from vllm.engine.arg_utils import EngineArgs from vllm.logger import init_logger +from vllm.transformers_utils.gguf_utils import is_gguf from vllm.v1.engine.async_llm import AsyncEngineArgs from vllm_omni.config import OmniModelConfig @@ -89,6 +91,19 @@ class OmniEngineArgs(EngineArgs): Returns: OmniModelConfig instance with all configuration fields set """ + # GGUF files need a specific model loader path in vLLM. + if is_gguf(self.model): + self.quantization = self.load_format = "gguf" + + if not envs.VLLM_ENABLE_V1_MULTIPROCESSING: + logger.warning( + "The global random seed is set to %d. Since " + "VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may " + "affect the random state of the Python process that " + "launched vLLM.", + self.seed, + ) + # register omni models to avoid model not found error self._ensure_omni_models_registered() @@ -215,6 +230,19 @@ class AsyncOmniEngineArgs(AsyncEngineArgs): Returns: OmniModelConfig instance with all configuration fields set """ + # GGUF files need a specific model loader path in vLLM. + if is_gguf(self.model): + self.quantization = self.load_format = "gguf" + + if not envs.VLLM_ENABLE_V1_MULTIPROCESSING: + logger.warning( + "The global random seed is set to %d. Since " + "VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may " + "affect the random state of the Python process that " + "launched vLLM.", + self.seed, + ) + # register omni models to avoid model not found error self._ensure_omni_models_registered() diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index ceb4aa2f2..c4db891a0 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -33,7 +33,13 @@ from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, ToolServer from vllm.entrypoints.openai.api_server import build_app as build_openai_app from vllm.entrypoints.openai.api_server import setup_server as setup_openai_server -from vllm.entrypoints.openai.basic.api_router import base + +# vLLM moved `base` from openai.basic.api_router to serve.instrumentator.basic. +# Keep a fallback for older/newer upstream layouts during rebase windows. +try: + from vllm.entrypoints.serve.instrumentator.basic import base +except ModuleNotFoundError: + from vllm.entrypoints.openai.basic.api_router import base from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponse,