[Update] Upgrade vLLM wheel version and enhance metrics handling
- Updated the vLLM wheel to commit 24650715105a26b41c88f4164777b5e1a0f3200b in Dockerfile.ci. - Modified the `calculate_metrics` function to handle `tokenizer` as optional (can be None) in metrics.py. - Adjusted the `benchmark` function to accept `tokenizer` as optional in patch.py. - Enhanced condition checks in OmniARScheduler to ensure valid token generation in omni_ar_scheduler.py. - Updated input processing to include supported tasks in input_processor.py. - Cleaned up unnecessary code in output_processor.py. - Added support for SSL ciphers in the API server configuration in api_server.py. - Enabled speculative decoding in GPU model runners for improved performance in gpu_generation_model_runner.py and gpu_model_runner.py. Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
22e01c416b
commit
a252918008
9 files changed
+30
-26
No files matched your search
@@ -14,11 +14,11 @@ RUN apt-get update && \
|
||||
|
||||
# Install vLLM wheel for the specific commit
|
||||
# URL format: https://wheels.vllm.ai/<commit_hash>/
|
||||
# Current vLLM wheel commit: 16f24e87975ef4cd2c12879425062913ef62f6fd
|
||||
# Current vLLM wheel commit: 24650715105a26b41c88f4164777b5e1a0f3200b
|
||||
# Note: --prerelease=allow is required because nightly/commit wheels use
|
||||
# pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default,
|
||||
# causing it to fall back to the older stable release from PyPI.
|
||||
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/ec85340531e83962877fa683e23f1d5d17cd786a
|
||||
RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/24650715105a26b41c88f4164777b5e1a0f3200b
|
||||
|
||||
|
||||
RUN uv pip uninstall --system nvidia-cublas-cu12
|
||||
|
||||
@@ -134,7 +134,7 @@ def calculate_metrics(
|
||||
input_requests: list[SampleRequest],
|
||||
outputs: list[RequestFuncOutput],
|
||||
dur_s: float,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
tokenizer: PreTrainedTokenizerBase | None,
|
||||
selected_percentiles: list[float],
|
||||
goodput_config_dict: dict[str, float],
|
||||
task_type,
|
||||
@@ -169,17 +169,21 @@ def calculate_metrics(
|
||||
audio_rtfs: list[float] = []
|
||||
audio_duration: list[float] = []
|
||||
audio_frames: list[int] = []
|
||||
input_audio_duration = 0.0
|
||||
for i in range(len(outputs)):
|
||||
if outputs[i].success:
|
||||
output_len = outputs[i].output_tokens
|
||||
|
||||
if not output_len:
|
||||
# We use the tokenizer to count the number of output tokens
|
||||
# for some serving backends instead of looking at
|
||||
# len(outputs[i].itl) since multiple output tokens may be
|
||||
# bundled together
|
||||
# Note : this may inflate the output token count slightly
|
||||
output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids)
|
||||
if tokenizer is None:
|
||||
output_len = 1
|
||||
else:
|
||||
# We use the tokenizer to count the number of output tokens
|
||||
# for some serving backends instead of looking at
|
||||
# len(outputs[i].itl) since multiple output tokens may be
|
||||
# bundled together
|
||||
# Note : this may inflate the output token count slightly
|
||||
output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids)
|
||||
actual_output_lens.append(output_len)
|
||||
total_input += input_requests[i].prompt_len
|
||||
tpot = 0
|
||||
@@ -196,6 +200,7 @@ def calculate_metrics(
|
||||
audio_duration.append(getattr(outputs[i], "audio_duration", 0.0))
|
||||
audio_frames.append(getattr(outputs[i], "audio_frames", 0.0))
|
||||
e2els.append(outputs[i].latency)
|
||||
input_audio_duration += outputs[i].input_audio_duration
|
||||
completed += 1
|
||||
else:
|
||||
actual_output_lens.append(0)
|
||||
@@ -335,6 +340,7 @@ def calculate_metrics(
|
||||
percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles],
|
||||
max_output_tokens_per_s=max_output_tokens_per_s,
|
||||
max_concurrent_requests=max_concurrent_requests,
|
||||
rtfx=input_audio_duration / dur_s,
|
||||
)
|
||||
print_metrics(
|
||||
task_type,
|
||||
|
||||
@@ -223,7 +223,7 @@ async def benchmark(
|
||||
base_url: str,
|
||||
model_id: str,
|
||||
model_name: str,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
tokenizer: PreTrainedTokenizerBase | None,
|
||||
input_requests: list[SampleRequest],
|
||||
logprobs: int | None,
|
||||
request_rate: float,
|
||||
|
||||
@@ -246,7 +246,7 @@ class OmniARScheduler(VLLMScheduler):
|
||||
generated_token_ids = sampled_token_ids[req_index] if sampled_token_ids else []
|
||||
|
||||
scheduled_spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id)
|
||||
if scheduled_spec_token_ids:
|
||||
if scheduled_spec_token_ids and generated_token_ids:
|
||||
num_draft_tokens = len(scheduled_spec_token_ids)
|
||||
num_accepted = len(generated_token_ids) - 1
|
||||
num_rejected = num_draft_tokens - num_accepted
|
||||
@@ -476,10 +476,7 @@ class OmniARScheduler(VLLMScheduler):
|
||||
assert request.is_finished()
|
||||
|
||||
# 1. Standard cleanup parts from base _free_request
|
||||
connector_delay_free_blocks = False
|
||||
kv_xfer_params = None
|
||||
if self.connector is not None:
|
||||
connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)
|
||||
connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)
|
||||
|
||||
self.encoder_cache_manager.free(request)
|
||||
request_id = request.request_id
|
||||
|
||||
@@ -121,13 +121,13 @@ class OmniInputProcessor(InputProcessor):
|
||||
request_id: str,
|
||||
prompt: PromptType | ProcessorInputs,
|
||||
params: SamplingParams | PoolingParams,
|
||||
supported_tasks: tuple[SupportedTask, ...] = ("generate",),
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
data_parallel_rank: int | None = None,
|
||||
supported_tasks: tuple[SupportedTask, ...] | None = None,
|
||||
resumable: bool = False,
|
||||
) -> OmniEngineCoreRequest:
|
||||
"""Process input prompt into an engine core request.
|
||||
@@ -141,6 +141,7 @@ class OmniInputProcessor(InputProcessor):
|
||||
request_id: Unique identifier for this request
|
||||
prompt: Input prompt (text, token IDs, embeddings, or multimodal)
|
||||
params: Sampling or pooling parameters for generation
|
||||
supported_tasks: Tuple of supported tasks for validation
|
||||
arrival_time: Optional arrival timestamp (defaults to current time)
|
||||
lora_request: Optional LoRA adapter request
|
||||
tokenization_kwargs: Optional additional tokenization arguments
|
||||
@@ -148,19 +149,17 @@ class OmniInputProcessor(InputProcessor):
|
||||
priority: Request priority (higher values processed first)
|
||||
data_parallel_rank: Optional data parallel rank for distributed
|
||||
inference
|
||||
resumable: Whether the request supports streaming input
|
||||
|
||||
Returns:
|
||||
Tuple of (prompt_string, OmniEngineCoreRequest) where:
|
||||
- prompt_string: The original prompt as a string, or None if
|
||||
using embeddings
|
||||
- OmniEngineCoreRequest: Processed request ready for the engine
|
||||
OmniEngineCoreRequest ready for the engine
|
||||
|
||||
Raises:
|
||||
ValueError: If data_parallel_rank is out of range or prompt_embeds
|
||||
has incorrect shape
|
||||
"""
|
||||
self._validate_lora(lora_request)
|
||||
self._validate_params(params, supported_tasks)
|
||||
self._validate_lora(lora_request)
|
||||
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
@@ -187,8 +186,6 @@ class OmniInputProcessor(InputProcessor):
|
||||
|
||||
self._platform_validate_request(processed_inputs, params)
|
||||
|
||||
eos_token_id = self.renderer.get_eos_token_id()
|
||||
|
||||
encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)
|
||||
self._validate_model_inputs(encoder_inputs, decoder_inputs)
|
||||
|
||||
@@ -209,7 +206,10 @@ class OmniInputProcessor(InputProcessor):
|
||||
if sampling_params.max_tokens is None:
|
||||
seq_len = length_from_prompt_token_ids_or_embeds(prompt_token_ids, prompt_embeds)
|
||||
sampling_params.max_tokens = self.model_config.max_model_len - seq_len
|
||||
sampling_params.update_from_generation_config(self.generation_config_fields, eos_token_id)
|
||||
sampling_params.update_from_generation_config(
|
||||
self.generation_config_fields,
|
||||
self.renderer.get_eos_token_id(),
|
||||
)
|
||||
if self.tokenizer is not None:
|
||||
sampling_params.update_from_tokenizer(self.tokenizer)
|
||||
else:
|
||||
|
||||
@@ -337,8 +337,6 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
log_stats=self.log_stats,
|
||||
stream_interval=self.stream_interval,
|
||||
)
|
||||
if self._requests_drained.is_set():
|
||||
self._requests_drained.clear()
|
||||
self.request_states[request_id] = req_state
|
||||
if parent_req:
|
||||
self.parent_requests[parent_req.request_id] = parent_req
|
||||
|
||||
@@ -257,6 +257,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
|
||||
@@ -646,6 +646,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices),
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
use_spec_decode=self.speculative_config is not None,
|
||||
)
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
|
||||
@@ -593,6 +593,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices),
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
use_spec_decode=self.speculative_config is not None,
|
||||
)
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
|
||||
Reference in new issue
Block a user