diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 3b90ac261..117aba153 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -17,7 +17,7 @@ RUN apt-get update && \ # Note: --prerelease=allow is required because nightly/commit wheels use # pre-release version numbers (e.g. 2.10.0.dev...) which uv ignores by default, # causing it to fall back to the older stable release from PyPI. -RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/097eb544e9a22810c9b7a59e586b61627b308362 +RUN uv pip install --system --upgrade --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/84e436ed1c94b1b94f809927b5d6bff45f7af919 RUN uv pip install --system --no-cache-dir ".[dev]" diff --git a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py index 84112336e..8e04b0496 100644 --- a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py +++ b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py @@ -123,6 +123,7 @@ def make_mock_model(hidden: int = 8): cfg.video_token_index = VIDEO_TOKEN_ID cfg.audio_token_index = AUDIO_TOKEN_ID model.config = cfg + model._has_oov_mm_tokens = False def fake_lm_embed(ids: torch.Tensor) -> torch.Tensor: # Use .clone() so the tensor is contiguous (expand() creates a strided @@ -137,13 +138,12 @@ def make_mock_model(hidden: int = 8): model._embed_text_input_ids = lambda *a, **kw: SupportsMultiModal._embed_text_input_ids(model, *a, **kw) - def fake_super_embed(ids, mm_embs=None, *, is_multimodal=None, handle_oov_mm_token=False): + def fake_super_embed(ids, mm_embs=None, *, is_multimodal=None): return SupportsMultiModal.embed_input_ids( model, ids, mm_embs, is_multimodal=is_multimodal, - handle_oov_mm_token=handle_oov_mm_token, ) model.embed_input_ids = lambda *a, **kw: Qwen2_5OmniThinkerForConditionalGeneration.embed_input_ids(model, *a, **kw) diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index a397e6085..b2e64224b 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -247,6 +247,15 @@ class OmniGenerationScheduler(VLLMScheduler): ) total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + + # Record the request ids scheduled in this step (v0.14.0 behavior). + self.prev_step_scheduled_req_ids.clear() + self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys()) + + new_block_ids_to_zero = ( + (self.kv_cache_manager.take_new_block_ids() or None) if self.needs_kv_cache_zeroing else None + ) + scheduler_output = SchedulerOutput( scheduled_new_reqs=new_reqs_data, scheduled_cached_reqs=cached_reqs_data, @@ -258,12 +267,9 @@ class OmniGenerationScheduler(VLLMScheduler): finished_req_ids=self.finished_req_ids, free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(), preempted_req_ids=set(), + new_block_ids_to_zero=new_block_ids_to_zero, ) - # Record the request ids scheduled in this step (v0.14.0 behavior). - self.prev_step_scheduled_req_ids.clear() - self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys()) - # KVTransfer: package metadata if self.connector is not None: meta = self.connector.build_connector_meta(scheduler_output) diff --git a/vllm_omni/engine/input_processor.py b/vllm_omni/engine/input_processor.py index 5bbd16b38..d785db2fd 100644 --- a/vllm_omni/engine/input_processor.py +++ b/vllm_omni/engine/input_processor.py @@ -11,6 +11,7 @@ from vllm.logger import init_logger from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.multimodal.utils import argsort_mm_positions +from vllm.platforms import current_platform from vllm.pooling_params import PoolingParams from vllm.renderers import BaseRenderer from vllm.sampling_params import SamplingParams @@ -184,7 +185,7 @@ class OmniInputProcessor(InputProcessor): tokenization_kwargs=tokenization_kwargs, ) - self._platform_validate_request(processed_inputs, params) + current_platform.validate_request(processed_inputs, params) encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs) self._validate_model_inputs(encoder_inputs, decoder_inputs) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 6db97dd2d..b6298c037 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -67,7 +67,7 @@ from vllm.entrypoints.openai.speech_to_text.serving import ( ) from vllm.entrypoints.openai.utils import validate_json_request from vllm.entrypoints.pooling.classify.serving import ServingClassification -from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding +from vllm.entrypoints.pooling.embed.serving import ServingEmbedding from vllm.entrypoints.pooling.pooling.serving import OpenAIServingPooling from vllm.entrypoints.pooling.score.serving import ServingScores from vllm.entrypoints.serve.disagg.serving import ServingTokens @@ -570,7 +570,6 @@ async def omni_init_app_state( enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, enable_log_outputs=args.enable_log_outputs, - log_error_stack=args.log_error_stack, ) if "generate" in supported_tasks else None @@ -594,14 +593,13 @@ async def omni_init_app_state( enable_force_include_usage=args.enable_force_include_usage, enable_log_outputs=args.enable_log_outputs, enable_log_deltas=args.enable_log_deltas, - log_error_stack=args.log_error_stack, ) if "generate" in supported_tasks else None ) # Warm up chat template processing to avoid first-request latency if state.openai_serving_chat is not None: - await state.openai_serving_chat.warmup() + state.openai_serving_chat.warmup() state.openai_serving_completion = ( OpenAIServingCompletion( @@ -611,7 +609,6 @@ async def omni_init_app_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, - log_error_stack=args.log_error_stack, ) if "generate" in supported_tasks else None @@ -625,20 +622,18 @@ async def omni_init_app_state( chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, trust_request_chat_template=args.trust_request_chat_template, - log_error_stack=args.log_error_stack, ) if any(task in POOLING_TASKS for task in supported_tasks) else None ) state.openai_serving_embedding = ( - OpenAIServingEmbedding( + ServingEmbedding( engine_client, state.openai_serving_models, request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, trust_request_chat_template=args.trust_request_chat_template, - log_error_stack=args.log_error_stack, ) if "embed" in supported_tasks else None @@ -651,7 +646,6 @@ async def omni_init_app_state( chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, trust_request_chat_template=args.trust_request_chat_template, - log_error_stack=args.log_error_stack, ) if "classify" in supported_tasks else None @@ -664,7 +658,7 @@ async def omni_init_app_state( score_template=resolved_chat_template, log_error_stack=args.log_error_stack, ) - if ("embed" in supported_tasks or "score" in supported_tasks) + if any(t in supported_tasks for t in ("embed", "score", "token_embed")) else None ) state.openai_serving_tokenization = OpenAIServingTokenization( @@ -674,14 +668,12 @@ async def omni_init_app_state( chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, trust_request_chat_template=args.trust_request_chat_template, - log_error_stack=args.log_error_stack, ) state.openai_serving_transcription = ( OpenAIServingTranscription( engine_client, state.openai_serving_models, request_logger=request_logger, - log_error_stack=args.log_error_stack, enable_force_include_usage=args.enable_force_include_usage, ) if "transcription" in supported_tasks @@ -692,7 +684,6 @@ async def omni_init_app_state( engine_client, state.openai_serving_models, request_logger=request_logger, - log_error_stack=args.log_error_stack, enable_force_include_usage=args.enable_force_include_usage, ) if "transcription" in supported_tasks @@ -722,7 +713,6 @@ async def omni_init_app_state( state.openai_serving_models, request_logger=request_logger, return_tokens_as_token_ids=args.return_tokens_as_token_ids, - log_error_stack=args.log_error_stack, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_log_outputs=args.enable_log_outputs, force_no_detokenize=args.tokens_only, diff --git a/vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py b/vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py index b015679c2..b886aec69 100644 --- a/vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py +++ b/vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py @@ -1328,7 +1328,6 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo multimodal_embeddings: MultiModalEmbeddings | None = None, *, is_multimodal: torch.Tensor | None = None, - handle_oov_mm_token: bool = False, ) -> torch.Tensor: """Embed input IDs with optional multimodal embeddings.""" # Get text embeddings diff --git a/vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py b/vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py index c1b71e076..1424ca775 100644 --- a/vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py +++ b/vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py @@ -773,7 +773,6 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor multimodal_embeddings: MultiModalEmbeddings | None = None, *, is_multimodal: torch.Tensor | None = None, - handle_oov_mm_token: bool = False, ) -> torch.Tensor: # This is to satisfy the type checker for each overload if multimodal_embeddings is None or is_multimodal is None: @@ -783,7 +782,6 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal, - handle_oov_mm_token=handle_oov_mm_token, ) def base_local_forward( diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py index 988d8c236..8c348a9c6 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py @@ -109,7 +109,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration( multimodal_embeddings: MultiModalEmbeddings | None = None, *, is_multimodal: torch.Tensor | None = None, - handle_oov_mm_token: bool = False, ) -> torch.Tensor: # This is to satisfy the type checker for each overload if multimodal_embeddings is None or is_multimodal is None: @@ -119,7 +118,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration( input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal, - handle_oov_mm_token=handle_oov_mm_token, ) def forward( diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py index 64eaefaa2..c32b121c2 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py @@ -586,11 +586,19 @@ class Qwen2_5OmniThinkerForConditionalGeneration( multimodal_embeddings: MultiModalEmbeddings | None = None, *, is_multimodal: torch.Tensor | None = None, - handle_oov_mm_token: bool = False, ) -> torch.Tensor: if multimodal_embeddings is None or is_multimodal is None: return super().embed_input_ids(input_ids) + inputs_embeds = self._embed_text_input_ids( + input_ids, + self.get_language_model().embed_input_ids, + is_multimodal=is_multimodal, + ) + + if len(multimodal_embeddings) == 0: + return inputs_embeds + # Check for audio-in-video: interleaved video and audio tokens # in the multimodal region. Only use the interleaved path when # needed; otherwise fall back to the default parent implementation. @@ -608,7 +616,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration( input_ids, self.get_language_model().embed_input_ids, is_multimodal=is_multimodal, - handle_oov_mm_token=handle_oov_mm_token, ) return merge_interleaved_embeddings( inputs_embeds, @@ -625,7 +632,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration( input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal, - handle_oov_mm_token=handle_oov_mm_token, ) def forward( diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py index 7dfb988eb..852d5572b 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py @@ -966,13 +966,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( multimodal_embeddings: MultiModalEmbeddings | None = None, *, is_multimodal: torch.Tensor | None = None, - handle_oov_mm_token: bool = False, ) -> torch.Tensor: inputs_embeds = self._embed_text_input_ids( input_ids, self.language_model.embed_input_ids, is_multimodal=is_multimodal, - handle_oov_mm_token=handle_oov_mm_token, ) if multimodal_embeddings is None or len(multimodal_embeddings) == 0: @@ -1064,7 +1062,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal, - handle_oov_mm_token=handle_oov_mm_token, ) def forward( diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index d7d45031a..9dce9778d 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -141,7 +141,7 @@ class GPUARModelRunner(OmniGPUModelRunner): # Update persistent batch states. self._update_states(scheduler_output) - if has_ec_transfer() and get_ec_transfer().is_producer: + if has_ec_transfer() and not get_ec_transfer().is_consumer: with self.maybe_get_ec_connector_output( scheduler_output, encoder_cache=self.encoder_cache, @@ -276,9 +276,9 @@ class GPUARModelRunner(OmniGPUModelRunner): # Run the model. # Use persistent buffers for CUDA graphs. - # When spec decode is enabled, delay clearing connector metadata - # until after draft model runs in sample_tokens. - clear_kv_metadata = self.speculative_config is None + # When spec decode is enabled, defer connector finalization + # (wait_for_save + clear metadata) until after draft model runs. + defer_kv_connector_finalize = self.speculative_config is not None with ( set_forward_context( attn_metadata, @@ -292,7 +292,8 @@ class GPUARModelRunner(OmniGPUModelRunner): ), record_function_or_nullcontext("gpu_model_runner: forward"), self.maybe_get_kv_connector_output( - scheduler_output, clear_metadata=clear_kv_metadata + scheduler_output, + defer_finalize=defer_kv_connector_finalize, ) as kv_connector_output, ): model_output = self._model_forward( @@ -533,11 +534,11 @@ class GPUARModelRunner(OmniGPUModelRunner): # tokens on the CPU, so they are run after bookkeeping. propose_draft_token_ids(valid_sampled_token_ids) - # Clear KV connector metadata after draft model runs (if spec decode). - # This was deferred from target model forward to allow draft model - # to also save its KV cache. + # Finalize KV connector (wait_for_save + clear metadata) after + # draft model runs. Deferred from target model forward to allow + # draft model to also save its KV cache. if self.speculative_config is not None: - self.clear_kv_connector_metadata() + self.finalize_kv_connector() with record_function_or_nullcontext("gpu_model_runner: eplb"): self.eplb_step() diff --git a/vllm_omni/worker/gpu_ar_worker.py b/vllm_omni/worker/gpu_ar_worker.py index 198d42c8a..8734524a2 100644 --- a/vllm_omni/worker/gpu_ar_worker.py +++ b/vllm_omni/worker/gpu_ar_worker.py @@ -79,7 +79,7 @@ class GPUARWorker(OmniWorkerMixin, OmniGPUWorkerBase): # Now take memory snapshot after NCCL is initialized gc.collect() - torch.cuda.empty_cache() + torch.accelerator.empty_cache() # take current memory snapshot self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device) diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index 05785d7aa..0dd304ed2 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -108,7 +108,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): if not scheduler_output.total_num_scheduled_tokens: return EMPTY_MODEL_RUNNER_OUTPUT - if has_ec_transfer() and get_ec_transfer().is_producer: + if has_ec_transfer() and not get_ec_transfer().is_consumer: with self.maybe_get_ec_connector_output( scheduler_output, encoder_cache=self.encoder_cache, @@ -265,9 +265,9 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): # Run the model. # Use persistent buffers for CUDA graphs. - # When spec decode is enabled, delay clearing connector metadata - # until after draft model runs in sample_tokens. - clear_kv_metadata = self.speculative_config is None + # When spec decode is enabled, defer connector finalization + # (wait_for_save + clear metadata) until after draft model runs. + defer_kv_connector_finalize = self.speculative_config is not None with ( set_forward_context( attn_metadata, @@ -281,7 +281,8 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): ), record_function_or_nullcontext("Forward"), self.maybe_get_kv_connector_output( - scheduler_output, clear_metadata=clear_kv_metadata + scheduler_output, + defer_finalize=defer_kv_connector_finalize, ) as kv_connector_output, ): outputs = self._run_generation_model( @@ -351,9 +352,10 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): ) = self.execute_model_state self.execute_model_state = None - # Clear KV connector metadata after draft model runs (if spec decode). + # Finalize KV connector (wait_for_save + clear metadata) after + # draft model runs. Deferred from target model forward. if self.speculative_config is not None: - self.clear_kv_connector_metadata() + self.finalize_kv_connector() pooler_output: list[object] = [] if isinstance(multimodal_outputs, torch.Tensor): @@ -476,6 +478,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): remove_lora: bool = True, is_graph_capturing: bool = False, num_active_loras: int = 0, + profile_seq_lens: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: """ Run a dummy forward pass to warm up/profile run or capture the @@ -500,6 +503,9 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): remove_lora: If False, dummy LoRAs are not destroyed after the run num_active_loras: Number of distinct active LoRAs to capture for. LoRA is activated when num_active_loras > 0. + profile_seq_lens: If provided, use this value for seq_lens instead + of max_query_len. Used to profile attention workspace that + scales with context length. """ mm_config = self.vllm_config.model_config.multimodal_config if mm_config and mm_config.mm_encoder_only: @@ -621,11 +627,13 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): # If force_attention is True, we always capture attention. # Otherwise, it only happens for cudagraph_runtime_mode=FULL. if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL: - if create_mixed_batch: + if profile_seq_lens is not None: + seq_lens = profile_seq_lens # type: ignore[assignment] + elif create_mixed_batch: # In the mixed batch mode (used for FI warmup), we use # shorter sequence lengths to run faster. # TODO(luka) better system for describing dummy batches - seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1] + seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1] # type: ignore[assignment] else: seq_lens = max_query_len # type: ignore[assignment] self.seq_lens.np[:num_reqs] = seq_lens diff --git a/vllm_omni/worker/gpu_generation_worker.py b/vllm_omni/worker/gpu_generation_worker.py index 7de9a7922..341797ebb 100644 --- a/vllm_omni/worker/gpu_generation_worker.py +++ b/vllm_omni/worker/gpu_generation_worker.py @@ -79,7 +79,7 @@ class GPUGenerationWorker(OmniWorkerMixin, OmniGPUWorkerBase): # Now take memory snapshot after NCCL is initialized gc.collect() - torch.cuda.empty_cache() + torch.accelerator.empty_cache() # take current memory snapshot self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device) diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index e17d52bdd..5a3d2f9ca 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -238,6 +238,7 @@ class OmniGPUModelRunner(GPUModelRunner): self.requests.pop(req_id, None) self.model_intermediate_buffer.pop(req_id, None) self.num_prompt_logprobs.pop(req_id, None) + self.late_interaction_runner.on_requests_finished(scheduler_output.finished_req_ids) # Remove the finished requests from the persistent batch. # NOTE(woosuk): There could be an edge case where finished_req_ids and # scheduled_req_ids overlap. This happens when a request is aborted and @@ -247,6 +248,11 @@ class OmniGPUModelRunner(GPUModelRunner): for req_id in scheduler_output.finished_req_ids: self.input_batch.remove_request(req_id) + # Zero GPU memory for freshly allocated cache blocks to prevent + # stale NaN/data from corrupting attention or SSM computation. + if scheduler_output.new_block_ids_to_zero: + self._zero_block_ids(scheduler_output.new_block_ids_to_zero) + # Free the cached encoder outputs. for mm_hash in scheduler_output.free_encoder_mm_hashes: self.encoder_cache.pop(mm_hash, None) @@ -314,6 +320,7 @@ class OmniGPUModelRunner(GPUModelRunner): lora_request=new_req_data.lora_request, ) self.requests[req_id] = req_state + self.late_interaction_runner.register_request(req_id, pooling_params) # If prompt embeddings are provided, decode and attach to inter_data try: @@ -399,13 +406,13 @@ class OmniGPUModelRunner(GPUModelRunner): # prev_num_draft_len is used in async scheduling mode with # spec decode. it indicates if need to update num_computed_tokens # of the request. for example: - # fist step: num_computed_tokens = 0, spec_tokens = [], + # first step: num_computed_tokens = 0, spec_tokens = [], # prev_num_draft_len = 0. # second step: num_computed_tokens = 100(prompt length), # spec_tokens = [a,b], prev_num_draft_len = 0. # third step: num_computed_tokens = 100 + 2, spec_tokens = [c,d], # prev_num_draft_len = 2. - # num_computed_tokens in first step and second step does't contain + # num_computed_tokens in first step and second step doesn't contain # the spec tokens length, but in third step it contains the # spec tokens length. we only need to update num_computed_tokens # when prev_num_draft_len > 0. @@ -532,6 +539,7 @@ class OmniGPUModelRunner(GPUModelRunner): remove_lora: bool = True, is_graph_capturing: bool = False, num_active_loras: int = 0, + profile_seq_lens: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: """ Run a dummy forward pass to warm up/profile run or capture the @@ -556,6 +564,9 @@ class OmniGPUModelRunner(GPUModelRunner): remove_lora: If False, dummy LoRAs are not destroyed after the run num_active_loras: Number of distinct active LoRAs to capture for. LoRA is activated when num_active_loras > 0. + profile_seq_lens: If provided, use this value for seq_lens instead + of max_query_len. Used to profile attention workspace that + scales with context length. """ mm_config = self.vllm_config.model_config.multimodal_config if mm_config and mm_config.mm_encoder_only: @@ -676,11 +687,13 @@ class OmniGPUModelRunner(GPUModelRunner): # If force_attention is True, we always capture attention. # Otherwise, it only happens for cudagraph_runtime_mode=FULL. if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL: - if create_mixed_batch: + if profile_seq_lens is not None: + seq_lens = profile_seq_lens # type: ignore[assignment] + elif create_mixed_batch: # In the mixed batch mode (used for FI warmup), we use # shorter sequence lengths to run faster. # TODO(luka) better system for describing dummy batches - seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1] + seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1] # type: ignore[assignment] else: seq_lens = max_query_len # type: ignore[assignment] self.seq_lens.np[:num_reqs] = seq_lens