diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index c10ed7770..059c0eb89 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -14,11 +14,11 @@ RUN apt-get update && \ # Install vLLM wheel for the specific commit # URL format: https://wheels.vllm.ai// -# Current vLLM wheel commit: 16f24e87975ef4cd2c12879425062913ef62f6fd +# Current vLLM wheel commit: 24650715105a26b41c88f4164777b5e1a0f3200b # Note: --prerelease=allow is required because nightly/commit wheels use # pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default, # causing it to fall back to the older stable release from PyPI. -RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/ec85340531e83962877fa683e23f1d5d17cd786a +RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/24650715105a26b41c88f4164777b5e1a0f3200b RUN uv pip uninstall --system nvidia-cublas-cu12 diff --git a/vllm_omni/benchmarks/metrics/metrics.py b/vllm_omni/benchmarks/metrics/metrics.py index 83671b713..bbd981a8e 100644 --- a/vllm_omni/benchmarks/metrics/metrics.py +++ b/vllm_omni/benchmarks/metrics/metrics.py @@ -134,7 +134,7 @@ def calculate_metrics( input_requests: list[SampleRequest], outputs: list[RequestFuncOutput], dur_s: float, - tokenizer: PreTrainedTokenizerBase, + tokenizer: PreTrainedTokenizerBase | None, selected_percentiles: list[float], goodput_config_dict: dict[str, float], task_type, @@ -169,17 +169,21 @@ def calculate_metrics( audio_rtfs: list[float] = [] audio_duration: list[float] = [] audio_frames: list[int] = [] + input_audio_duration = 0.0 for i in range(len(outputs)): if outputs[i].success: output_len = outputs[i].output_tokens if not output_len: - # We use the tokenizer to count the number of output tokens - # for some serving backends instead of looking at - # len(outputs[i].itl) since multiple output tokens may be - # bundled together - # Note : this may inflate the output token count slightly - output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids) + if tokenizer is None: + output_len = 1 + else: + # We use the tokenizer to count the number of output tokens + # for some serving backends instead of looking at + # len(outputs[i].itl) since multiple output tokens may be + # bundled together + # Note : this may inflate the output token count slightly + output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids) actual_output_lens.append(output_len) total_input += input_requests[i].prompt_len tpot = 0 @@ -196,6 +200,7 @@ def calculate_metrics( audio_duration.append(getattr(outputs[i], "audio_duration", 0.0)) audio_frames.append(getattr(outputs[i], "audio_frames", 0.0)) e2els.append(outputs[i].latency) + input_audio_duration += outputs[i].input_audio_duration completed += 1 else: actual_output_lens.append(0) @@ -335,6 +340,7 @@ def calculate_metrics( percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles], max_output_tokens_per_s=max_output_tokens_per_s, max_concurrent_requests=max_concurrent_requests, + rtfx=input_audio_duration / dur_s, ) print_metrics( task_type, diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index f89fbe035..7ab0a2a0d 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -223,7 +223,7 @@ async def benchmark( base_url: str, model_id: str, model_name: str, - tokenizer: PreTrainedTokenizerBase, + tokenizer: PreTrainedTokenizerBase | None, input_requests: list[SampleRequest], logprobs: int | None, request_rate: float, diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 7b0a1e54d..996b94483 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -246,7 +246,7 @@ class OmniARScheduler(VLLMScheduler): generated_token_ids = sampled_token_ids[req_index] if sampled_token_ids else [] scheduled_spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id) - if scheduled_spec_token_ids: + if scheduled_spec_token_ids and generated_token_ids: num_draft_tokens = len(scheduled_spec_token_ids) num_accepted = len(generated_token_ids) - 1 num_rejected = num_draft_tokens - num_accepted @@ -476,10 +476,7 @@ class OmniARScheduler(VLLMScheduler): assert request.is_finished() # 1. Standard cleanup parts from base _free_request - connector_delay_free_blocks = False - kv_xfer_params = None - if self.connector is not None: - connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request) + connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request) self.encoder_cache_manager.free(request) request_id = request.request_id diff --git a/vllm_omni/engine/input_processor.py b/vllm_omni/engine/input_processor.py index e5c45ba6e..5bbd16b38 100644 --- a/vllm_omni/engine/input_processor.py +++ b/vllm_omni/engine/input_processor.py @@ -121,13 +121,13 @@ class OmniInputProcessor(InputProcessor): request_id: str, prompt: PromptType | ProcessorInputs, params: SamplingParams | PoolingParams, + supported_tasks: tuple[SupportedTask, ...] = ("generate",), arrival_time: float | None = None, lora_request: LoRARequest | None = None, tokenization_kwargs: dict[str, Any] | None = None, trace_headers: Mapping[str, str] | None = None, priority: int = 0, data_parallel_rank: int | None = None, - supported_tasks: tuple[SupportedTask, ...] | None = None, resumable: bool = False, ) -> OmniEngineCoreRequest: """Process input prompt into an engine core request. @@ -141,6 +141,7 @@ class OmniInputProcessor(InputProcessor): request_id: Unique identifier for this request prompt: Input prompt (text, token IDs, embeddings, or multimodal) params: Sampling or pooling parameters for generation + supported_tasks: Tuple of supported tasks for validation arrival_time: Optional arrival timestamp (defaults to current time) lora_request: Optional LoRA adapter request tokenization_kwargs: Optional additional tokenization arguments @@ -148,19 +149,17 @@ class OmniInputProcessor(InputProcessor): priority: Request priority (higher values processed first) data_parallel_rank: Optional data parallel rank for distributed inference + resumable: Whether the request supports streaming input Returns: - Tuple of (prompt_string, OmniEngineCoreRequest) where: - - prompt_string: The original prompt as a string, or None if - using embeddings - - OmniEngineCoreRequest: Processed request ready for the engine + OmniEngineCoreRequest ready for the engine Raises: ValueError: If data_parallel_rank is out of range or prompt_embeds has incorrect shape """ - self._validate_lora(lora_request) self._validate_params(params, supported_tasks) + self._validate_lora(lora_request) parallel_config = self.vllm_config.parallel_config dp_size = parallel_config.data_parallel_size @@ -187,8 +186,6 @@ class OmniInputProcessor(InputProcessor): self._platform_validate_request(processed_inputs, params) - eos_token_id = self.renderer.get_eos_token_id() - encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs) self._validate_model_inputs(encoder_inputs, decoder_inputs) @@ -209,7 +206,10 @@ class OmniInputProcessor(InputProcessor): if sampling_params.max_tokens is None: seq_len = length_from_prompt_token_ids_or_embeds(prompt_token_ids, prompt_embeds) sampling_params.max_tokens = self.model_config.max_model_len - seq_len - sampling_params.update_from_generation_config(self.generation_config_fields, eos_token_id) + sampling_params.update_from_generation_config( + self.generation_config_fields, + self.renderer.get_eos_token_id(), + ) if self.tokenizer is not None: sampling_params.update_from_tokenizer(self.tokenizer) else: diff --git a/vllm_omni/engine/output_processor.py b/vllm_omni/engine/output_processor.py index 8aa8ad9f4..b50e50d1e 100644 --- a/vllm_omni/engine/output_processor.py +++ b/vllm_omni/engine/output_processor.py @@ -337,8 +337,6 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): log_stats=self.log_stats, stream_interval=self.stream_interval, ) - if self._requests_drained.is_set(): - self._requests_drained.clear() self.request_states[request_id] = req_state if parent_req: self.parent_requests[parent_req.request_id] = parent_req diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 3898da308..5c4b73a18 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -257,6 +257,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None, ssl_certfile=args.ssl_certfile, ssl_ca_certs=args.ssl_ca_certs, ssl_cert_reqs=args.ssl_cert_reqs, + ssl_ciphers=args.ssl_ciphers, h11_max_incomplete_event_size=args.h11_max_incomplete_event_size, h11_max_header_count=args.h11_max_header_count, **uvicorn_kwargs, diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index a2c042839..4d7bfe2c4 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -646,6 +646,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices), for_cudagraph_capture=is_graph_capturing, slot_mappings=slot_mappings_by_group, + use_spec_decode=self.speculative_config is not None, ) with self.maybe_dummy_run_with_lora( diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index bafc6f963..76cd63654 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -593,6 +593,7 @@ class OmniGPUModelRunner(GPUModelRunner): ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices), for_cudagraph_capture=is_graph_capturing, slot_mappings=slot_mappings_by_group, + use_spec_decode=self.speculative_config is not None, ) with self.maybe_dummy_run_with_lora(