1 parent
3e539fcb6a
commit
98926707f5
5 files changed
+45
-15
No files matched your search
@@ -14,11 +14,11 @@ RUN apt-get update && \
|
||||
|
||||
# Install vLLM wheel for the specific commit
|
||||
# URL format: https://wheels.vllm.ai/<commit_hash>/
|
||||
# Current vLLM wheel commit: a60985b07eaf0fefa4c353ad2309b745707591ed
|
||||
# Current vLLM wheel commit: 6521ccf2860eb72cf23a9fa9044bf9721fd9a7c6
|
||||
# Note: --prerelease=allow is required because nightly/commit wheels use
|
||||
# pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default,
|
||||
# causing it to fall back to the older stable release from PyPI.
|
||||
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/a60985b07eaf0fefa4c353ad2309b745707591ed
|
||||
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/6521ccf2860eb72cf23a9fa9044bf9721fd9a7c6
|
||||
|
||||
RUN uv pip install --system --no-cache-dir ".[dev]"
|
||||
RUN uv pip install --system --upgrade "flashinfer-cubin==0.6.4"
|
||||
|
||||
@@ -1289,11 +1289,21 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
latest_delta_len = len(delta_message.tool_calls[0].function.arguments)
|
||||
|
||||
# get the expected call based on partial JSON
|
||||
# parsing which "autocompletes" the JSON
|
||||
expected_call = json.dumps(
|
||||
tool_parser.prev_tool_call_arr[index].get("arguments", {}),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
# parsing which "autocompletes" the JSON.
|
||||
# Tool parsers (e.g. Qwen3Coder) store
|
||||
# arguments as a JSON string in
|
||||
# prev_tool_call_arr. Calling json.dumps()
|
||||
# on an already-serialized string would
|
||||
# double-serialize it (e.g. '{"k":1}' becomes
|
||||
# '"{\\"k\\":1}"'), which then causes the
|
||||
# replace() below to fail and append the
|
||||
# entire double-serialized string as a
|
||||
# spurious final delta.
|
||||
args = tool_parser.prev_tool_call_arr[index].get("arguments", {})
|
||||
if isinstance(args, str):
|
||||
expected_call = args
|
||||
else:
|
||||
expected_call = json.dumps(args, ensure_ascii=False)
|
||||
|
||||
# get what we've streamed so far for arguments
|
||||
# for the current tool
|
||||
|
||||
@@ -22,7 +22,9 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
)
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.structured_output.utils import apply_grammar_bitmask
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm.v1.worker.gpu_model_runner import (
|
||||
@@ -459,10 +461,14 @@ class GPUARModelRunner(OmniGPUModelRunner):
|
||||
spec_decode_common_attn_metadata.max_seq_len + self.num_spec_tokens
|
||||
<= self.effective_drafter_max_model_len
|
||||
)
|
||||
if spec_config.use_eagle() and not spec_config.disable_padded_drafter_batch:
|
||||
# EAGLE speculative decoding can use the GPU sampled tokens
|
||||
# as inputs, and does not need to wait for bookkeeping to finish.
|
||||
assert isinstance(self.drafter, EagleProposer)
|
||||
use_gpu_toks = (
|
||||
spec_config.use_eagle() or spec_config.uses_draft_model() or spec_config.uses_extract_hidden_states()
|
||||
) and not spec_config.disable_padded_drafter_batch
|
||||
if use_gpu_toks:
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer,
|
||||
)
|
||||
sampled_token_ids = sampler_output.sampled_token_ids
|
||||
if input_fits_in_drafter:
|
||||
propose_draft_token_ids(sampled_token_ids)
|
||||
|
||||
@@ -23,7 +23,9 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm.v1.worker.gpu_model_runner import (
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
@@ -725,9 +727,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
hidden_states = outputs
|
||||
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states)
|
||||
if self.speculative_config and (
|
||||
self.speculative_config.use_eagle() or self.speculative_config.uses_draft_model()
|
||||
self.speculative_config.use_eagle()
|
||||
or self.speculative_config.uses_draft_model()
|
||||
or self.speculative_config.uses_extract_hidden_states()
|
||||
):
|
||||
assert isinstance(self.drafter, EagleProposer)
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer,
|
||||
)
|
||||
assert self.speculative_config is not None
|
||||
# Eagle currently only supports PIECEWISE cudagraphs.
|
||||
# Therefore only use cudagraphs if the main model uses PIECEWISE
|
||||
|
||||
@@ -14,7 +14,9 @@ from vllm.sampling_params import SamplingType
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner, IntermediateTensors, PerLayerAttnMetadata
|
||||
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
|
||||
@@ -796,9 +798,14 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
hidden_states = outputs
|
||||
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states)
|
||||
if self.speculative_config and (
|
||||
self.speculative_config.use_eagle() or self.speculative_config.uses_draft_model()
|
||||
self.speculative_config.use_eagle()
|
||||
or self.speculative_config.uses_draft_model()
|
||||
or self.speculative_config.uses_extract_hidden_states()
|
||||
):
|
||||
assert isinstance(self.drafter, EagleProposer)
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer,
|
||||
)
|
||||
assert self.speculative_config is not None
|
||||
# Eagle currently only supports PIECEWISE cudagraphs.
|
||||
# Therefore only use cudagraphs if the main model uses PIECEWISE
|
||||
|
||||
Reference in new issue
Block a user