[Update] Upgrade vLLM wheel version and enhance metrics handling

- Updated the vLLM wheel to commit 24650715105a26b41c88f4164777b5e1a0f3200b in Dockerfile.ci.
- Modified the `calculate_metrics` function to handle `tokenizer` as optional (can be None) in metrics.py.
- Adjusted the `benchmark` function to accept `tokenizer` as optional in patch.py.
- Enhanced condition checks in OmniARScheduler to ensure valid token generation in omni_ar_scheduler.py.
- Updated input processing to include supported tasks in input_processor.py.
- Cleaned up unnecessary code in output_processor.py.
- Added support for SSL ciphers in the API server configuration in api_server.py.
- Enabled speculative decoding in GPU model runners for improved performance in gpu_generation_model_runner.py and gpu_model_runner.py.

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-02-25 10:02:23 +00:00
1 parent 22e01c416b
commit a252918008
9 files changed
+30 -26

No files matched your search

+2 -2
View File
@@ -14,11 +14,11 @@ RUN apt-get update && \
# Install vLLM wheel for the specific commit
# URL format: https://wheels.vllm.ai/<commit_hash>/
# Current vLLM wheel commit: 16f24e87975ef4cd2c12879425062913ef62f6fd
# Current vLLM wheel commit: 24650715105a26b41c88f4164777b5e1a0f3200b
# Note: --prerelease=allow is required because nightly/commit wheels use
# pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default,
# causing it to fall back to the older stable release from PyPI.
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/ec85340531e83962877fa683e23f1d5d17cd786a
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/24650715105a26b41c88f4164777b5e1a0f3200b
RUN uv pip uninstall --system nvidia-cublas-cu12
+13 -7
View File
@@ -134,7 +134,7 @@ def calculate_metrics(
input_requests: list[SampleRequest],
outputs: list[RequestFuncOutput],
dur_s: float,
tokenizer: PreTrainedTokenizerBase,
tokenizer: PreTrainedTokenizerBase | None,
selected_percentiles: list[float],
goodput_config_dict: dict[str, float],
task_type,
@@ -169,17 +169,21 @@ def calculate_metrics(
audio_rtfs: list[float] = []
audio_duration: list[float] = []
audio_frames: list[int] = []
input_audio_duration = 0.0
for i in range(len(outputs)):
if outputs[i].success:
output_len = outputs[i].output_tokens
if not output_len:
# We use the tokenizer to count the number of output tokens
# for some serving backends instead of looking at
# len(outputs[i].itl) since multiple output tokens may be
# bundled together
# Note : this may inflate the output token count slightly
output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids)
if tokenizer is None:
output_len = 1
else:
# We use the tokenizer to count the number of output tokens
# for some serving backends instead of looking at
# len(outputs[i].itl) since multiple output tokens may be
# bundled together
# Note : this may inflate the output token count slightly
output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids)
actual_output_lens.append(output_len)
total_input += input_requests[i].prompt_len
tpot = 0
@@ -196,6 +200,7 @@ def calculate_metrics(
audio_duration.append(getattr(outputs[i], "audio_duration", 0.0))
audio_frames.append(getattr(outputs[i], "audio_frames", 0.0))
e2els.append(outputs[i].latency)
input_audio_duration += outputs[i].input_audio_duration
completed += 1
else:
actual_output_lens.append(0)
@@ -335,6 +340,7 @@ def calculate_metrics(
percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles],
max_output_tokens_per_s=max_output_tokens_per_s,
max_concurrent_requests=max_concurrent_requests,
rtfx=input_audio_duration / dur_s,
)
print_metrics(
task_type,
+1 -1
View File
@@ -223,7 +223,7 @@ async def benchmark(
base_url: str,
model_id: str,
model_name: str,
tokenizer: PreTrainedTokenizerBase,
tokenizer: PreTrainedTokenizerBase | None,
input_requests: list[SampleRequest],
logprobs: int | None,
request_rate: float,
+2 -5
View File
@@ -246,7 +246,7 @@ class OmniARScheduler(VLLMScheduler):
generated_token_ids = sampled_token_ids[req_index] if sampled_token_ids else []
scheduled_spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id)
if scheduled_spec_token_ids:
if scheduled_spec_token_ids and generated_token_ids:
num_draft_tokens = len(scheduled_spec_token_ids)
num_accepted = len(generated_token_ids) - 1
num_rejected = num_draft_tokens - num_accepted
@@ -476,10 +476,7 @@ class OmniARScheduler(VLLMScheduler):
assert request.is_finished()
# 1. Standard cleanup parts from base _free_request
connector_delay_free_blocks = False
kv_xfer_params = None
if self.connector is not None:
connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)
connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)
self.encoder_cache_manager.free(request)
request_id = request.request_id
+9 -9
View File
@@ -121,13 +121,13 @@ class OmniInputProcessor(InputProcessor):
request_id: str,
prompt: PromptType | ProcessorInputs,
params: SamplingParams | PoolingParams,
supported_tasks: tuple[SupportedTask, ...] = ("generate",),
arrival_time: float | None = None,
lora_request: LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
data_parallel_rank: int | None = None,
supported_tasks: tuple[SupportedTask, ...] | None = None,
resumable: bool = False,
) -> OmniEngineCoreRequest:
"""Process input prompt into an engine core request.
@@ -141,6 +141,7 @@ class OmniInputProcessor(InputProcessor):
request_id: Unique identifier for this request
prompt: Input prompt (text, token IDs, embeddings, or multimodal)
params: Sampling or pooling parameters for generation
supported_tasks: Tuple of supported tasks for validation
arrival_time: Optional arrival timestamp (defaults to current time)
lora_request: Optional LoRA adapter request
tokenization_kwargs: Optional additional tokenization arguments
@@ -148,19 +149,17 @@ class OmniInputProcessor(InputProcessor):
priority: Request priority (higher values processed first)
data_parallel_rank: Optional data parallel rank for distributed
inference
resumable: Whether the request supports streaming input
Returns:
Tuple of (prompt_string, OmniEngineCoreRequest) where:
- prompt_string: The original prompt as a string, or None if
using embeddings
- OmniEngineCoreRequest: Processed request ready for the engine
OmniEngineCoreRequest ready for the engine
Raises:
ValueError: If data_parallel_rank is out of range or prompt_embeds
has incorrect shape
"""
self._validate_lora(lora_request)
self._validate_params(params, supported_tasks)
self._validate_lora(lora_request)
parallel_config = self.vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
@@ -187,8 +186,6 @@ class OmniInputProcessor(InputProcessor):
self._platform_validate_request(processed_inputs, params)
eos_token_id = self.renderer.get_eos_token_id()
encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)
self._validate_model_inputs(encoder_inputs, decoder_inputs)
@@ -209,7 +206,10 @@ class OmniInputProcessor(InputProcessor):
if sampling_params.max_tokens is None:
seq_len = length_from_prompt_token_ids_or_embeds(prompt_token_ids, prompt_embeds)
sampling_params.max_tokens = self.model_config.max_model_len - seq_len
sampling_params.update_from_generation_config(self.generation_config_fields, eos_token_id)
sampling_params.update_from_generation_config(
self.generation_config_fields,
self.renderer.get_eos_token_id(),
)
if self.tokenizer is not None:
sampling_params.update_from_tokenizer(self.tokenizer)
else:
-2
View File
@@ -337,8 +337,6 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
log_stats=self.log_stats,
stream_interval=self.stream_interval,
)
if self._requests_drained.is_set():
self._requests_drained.clear()
self.request_states[request_id] = req_state
if parent_req:
self.parent_requests[parent_req.request_id] = parent_req
@@ -257,6 +257,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
ssl_certfile=args.ssl_certfile,
ssl_ca_certs=args.ssl_ca_certs,
ssl_cert_reqs=args.ssl_cert_reqs,
ssl_ciphers=args.ssl_ciphers,
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
h11_max_header_count=args.h11_max_header_count,
**uvicorn_kwargs,
@@ -646,6 +646,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices),
for_cudagraph_capture=is_graph_capturing,
slot_mappings=slot_mappings_by_group,
use_spec_decode=self.speculative_config is not None,
)
with self.maybe_dummy_run_with_lora(
+1
View File
@@ -593,6 +593,7 @@ class OmniGPUModelRunner(GPUModelRunner):
ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices),
for_cudagraph_capture=is_graph_capturing,
slot_mappings=slot_mappings_by_group,
use_spec_decode=self.speculative_config is not None,
)
with self.maybe_dummy_run_with_lora(