diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 45c99f2ad..db895e1bd 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -14,11 +14,11 @@ RUN apt-get update && \ # Install vLLM wheel for the specific commit # URL format: https://wheels.vllm.ai// -# Current vLLM wheel commit: a60985b07eaf0fefa4c353ad2309b745707591ed +# Current vLLM wheel commit: 6521ccf2860eb72cf23a9fa9044bf9721fd9a7c6 # Note: --prerelease=allow is required because nightly/commit wheels use # pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default, # causing it to fall back to the older stable release from PyPI. -RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/a60985b07eaf0fefa4c353ad2309b745707591ed +RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/6521ccf2860eb72cf23a9fa9044bf9721fd9a7c6 RUN uv pip install --system --no-cache-dir ".[dev]" RUN uv pip install --system --upgrade "flashinfer-cubin==0.6.4" diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index 71bb618ca..88a5378e2 100644 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -1289,11 +1289,21 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): latest_delta_len = len(delta_message.tool_calls[0].function.arguments) # get the expected call based on partial JSON - # parsing which "autocompletes" the JSON - expected_call = json.dumps( - tool_parser.prev_tool_call_arr[index].get("arguments", {}), - ensure_ascii=False, - ) + # parsing which "autocompletes" the JSON. + # Tool parsers (e.g. Qwen3Coder) store + # arguments as a JSON string in + # prev_tool_call_arr. Calling json.dumps() + # on an already-serialized string would + # double-serialize it (e.g. '{"k":1}' becomes + # '"{\\"k\\":1}"'), which then causes the + # replace() below to fail and append the + # entire double-serialized string as a + # spurious final delta. + args = tool_parser.prev_tool_call_arr[index].get("arguments", {}) + if isinstance(args, str): + expected_call = args + else: + expected_call = json.dumps(args, ensure_ascii=False) # get what we've streamed so far for arguments # for the current tool diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index 40a21a59c..e423101f0 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -22,7 +22,9 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( ) from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output +from vllm.v1.spec_decode.draft_model import DraftModelProposer from vllm.v1.spec_decode.eagle import EagleProposer +from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer from vllm.v1.structured_output.utils import apply_grammar_bitmask from vllm.v1.utils import record_function_or_nullcontext from vllm.v1.worker.gpu_model_runner import ( @@ -459,10 +461,14 @@ class GPUARModelRunner(OmniGPUModelRunner): spec_decode_common_attn_metadata.max_seq_len + self.num_spec_tokens <= self.effective_drafter_max_model_len ) - if spec_config.use_eagle() and not spec_config.disable_padded_drafter_batch: - # EAGLE speculative decoding can use the GPU sampled tokens - # as inputs, and does not need to wait for bookkeeping to finish. - assert isinstance(self.drafter, EagleProposer) + use_gpu_toks = ( + spec_config.use_eagle() or spec_config.uses_draft_model() or spec_config.uses_extract_hidden_states() + ) and not spec_config.disable_padded_drafter_batch + if use_gpu_toks: + assert isinstance( + self.drafter, + EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer, + ) sampled_token_ids = sampler_output.sampled_token_ids if input_fits_in_drafter: propose_draft_token_ids(sampled_token_ids) diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index 7d6f27597..fbbde44e1 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -23,7 +23,9 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( from vllm.utils.math_utils import cdiv from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output +from vllm.v1.spec_decode.draft_model import DraftModelProposer from vllm.v1.spec_decode.eagle import EagleProposer +from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer from vllm.v1.utils import record_function_or_nullcontext from vllm.v1.worker.gpu_model_runner import ( EMPTY_MODEL_RUNNER_OUTPUT, @@ -725,9 +727,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): hidden_states = outputs hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states) if self.speculative_config and ( - self.speculative_config.use_eagle() or self.speculative_config.uses_draft_model() + self.speculative_config.use_eagle() + or self.speculative_config.uses_draft_model() + or self.speculative_config.uses_extract_hidden_states() ): - assert isinstance(self.drafter, EagleProposer) + assert isinstance( + self.drafter, + EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer, + ) assert self.speculative_config is not None # Eagle currently only supports PIECEWISE cudagraphs. # Therefore only use cudagraphs if the main model uses PIECEWISE diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index 79181a0ed..509ae9875 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -14,7 +14,9 @@ from vllm.sampling_params import SamplingType from vllm.tracing import instrument from vllm.utils.import_utils import LazyLoader from vllm.utils.math_utils import cdiv +from vllm.v1.spec_decode.draft_model import DraftModelProposer from vllm.v1.spec_decode.eagle import EagleProposer +from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer from vllm.v1.worker.gpu_input_batch import CachedRequestState from vllm.v1.worker.gpu_model_runner import GPUModelRunner, IntermediateTensors, PerLayerAttnMetadata from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices @@ -796,9 +798,14 @@ class OmniGPUModelRunner(GPUModelRunner): hidden_states = outputs hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states) if self.speculative_config and ( - self.speculative_config.use_eagle() or self.speculative_config.uses_draft_model() + self.speculative_config.use_eagle() + or self.speculative_config.uses_draft_model() + or self.speculative_config.uses_extract_hidden_states() ): - assert isinstance(self.drafter, EagleProposer) + assert isinstance( + self.drafter, + EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer, + ) assert self.speculative_config is not None # Eagle currently only supports PIECEWISE cudagraphs. # Therefore only use cudagraphs if the main model uses PIECEWISE