rebase 2026/3/3

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-03-03 03:50:34 +00:00
1 parent 3e539fcb6a
commit 98926707f5
5 files changed
+45 -15

No files matched your search

+2 -2
View File
@@ -14,11 +14,11 @@ RUN apt-get update && \
# Install vLLM wheel for the specific commit
# URL format: https://wheels.vllm.ai/<commit_hash>/
# Current vLLM wheel commit: a60985b07eaf0fefa4c353ad2309b745707591ed
# Current vLLM wheel commit: 6521ccf2860eb72cf23a9fa9044bf9721fd9a7c6
# Note: --prerelease=allow is required because nightly/commit wheels use
# pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default,
# causing it to fall back to the older stable release from PyPI.
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/a60985b07eaf0fefa4c353ad2309b745707591ed
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/6521ccf2860eb72cf23a9fa9044bf9721fd9a7c6
RUN uv pip install --system --no-cache-dir ".[dev]"
RUN uv pip install --system --upgrade "flashinfer-cubin==0.6.4"
+15 -5
View File
@@ -1289,11 +1289,21 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
latest_delta_len = len(delta_message.tool_calls[0].function.arguments)
# get the expected call based on partial JSON
# parsing which "autocompletes" the JSON
expected_call = json.dumps(
tool_parser.prev_tool_call_arr[index].get("arguments", {}),
ensure_ascii=False,
)
# parsing which "autocompletes" the JSON.
# Tool parsers (e.g. Qwen3Coder) store
# arguments as a JSON string in
# prev_tool_call_arr. Calling json.dumps()
# on an already-serialized string would
# double-serialize it (e.g. '{"k":1}' becomes
# '"{\\"k\\":1}"'), which then causes the
# replace() below to fail and append the
# entire double-serialized string as a
# spurious final delta.
args = tool_parser.prev_tool_call_arr[index].get("arguments", {})
if isinstance(args, str):
expected_call = args
else:
expected_call = json.dumps(args, ensure_ascii=False)
# get what we've streamed so far for arguments
# for the current tool
+10 -4
View File
@@ -22,7 +22,9 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
)
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
from vllm.v1.structured_output.utils import apply_grammar_bitmask
from vllm.v1.utils import record_function_or_nullcontext
from vllm.v1.worker.gpu_model_runner import (
@@ -459,10 +461,14 @@ class GPUARModelRunner(OmniGPUModelRunner):
spec_decode_common_attn_metadata.max_seq_len + self.num_spec_tokens
<= self.effective_drafter_max_model_len
)
if spec_config.use_eagle() and not spec_config.disable_padded_drafter_batch:
# EAGLE speculative decoding can use the GPU sampled tokens
# as inputs, and does not need to wait for bookkeeping to finish.
assert isinstance(self.drafter, EagleProposer)
use_gpu_toks = (
spec_config.use_eagle() or spec_config.uses_draft_model() or spec_config.uses_extract_hidden_states()
) and not spec_config.disable_padded_drafter_batch
if use_gpu_toks:
assert isinstance(
self.drafter,
EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer,
)
sampled_token_ids = sampler_output.sampled_token_ids
if input_fits_in_drafter:
propose_draft_token_ids(sampled_token_ids)
@@ -23,7 +23,9 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
from vllm.utils.math_utils import cdiv
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
from vllm.v1.utils import record_function_or_nullcontext
from vllm.v1.worker.gpu_model_runner import (
EMPTY_MODEL_RUNNER_OUTPUT,
@@ -725,9 +727,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
hidden_states = outputs
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states)
if self.speculative_config and (
self.speculative_config.use_eagle() or self.speculative_config.uses_draft_model()
self.speculative_config.use_eagle()
or self.speculative_config.uses_draft_model()
or self.speculative_config.uses_extract_hidden_states()
):
assert isinstance(self.drafter, EagleProposer)
assert isinstance(
self.drafter,
EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer,
)
assert self.speculative_config is not None
# Eagle currently only supports PIECEWISE cudagraphs.
# Therefore only use cudagraphs if the main model uses PIECEWISE
+9 -2
View File
@@ -14,7 +14,9 @@ from vllm.sampling_params import SamplingType
from vllm.tracing import instrument
from vllm.utils.import_utils import LazyLoader
from vllm.utils.math_utils import cdiv
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
from vllm.v1.worker.gpu_input_batch import CachedRequestState
from vllm.v1.worker.gpu_model_runner import GPUModelRunner, IntermediateTensors, PerLayerAttnMetadata
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
@@ -796,9 +798,14 @@ class OmniGPUModelRunner(GPUModelRunner):
hidden_states = outputs
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states)
if self.speculative_config and (
self.speculative_config.use_eagle() or self.speculative_config.uses_draft_model()
self.speculative_config.use_eagle()
or self.speculative_config.uses_draft_model()
or self.speculative_config.uses_extract_hidden_states()
):
assert isinstance(self.drafter, EagleProposer)
assert isinstance(
self.drafter,
EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer,
)
assert self.speculative_config is not None
# Eagle currently only supports PIECEWISE cudagraphs.
# Therefore only use cudagraphs if the main model uses PIECEWISE