[CI] fix ci and format existing code (#71)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Jiangyun Zhu authored and GitHub committed 2025-11-18 20:54:26 +08:00
1 parent 09fab2ecc9
commit 5707fc78d5
52 files changed
+813 -2155

No files matched your search

+27 -72
View File
@@ -1,8 +1,7 @@
from typing import TYPE_CHECKING, List, Optional, Union, cast
from typing import TYPE_CHECKING, Optional, Union, cast
import numpy as np
import torch
from vllm.config import CUDAGraphMode
from vllm.distributed.parallel_state import get_pp_group
from vllm.forward_context import BatchDescriptor, set_forward_context
@@ -10,10 +9,7 @@ from vllm.logger import init_logger
from vllm.model_executor.models.interfaces_base import VllmModelForPooling
from vllm.sampling_params import SamplingType
from vllm.utils import LazyLoader, cdiv
from vllm.v1.attention.backends.utils import (
CommonAttentionMetadata,
split_attn_metadata,
)
from vllm.v1.attention.backends.utils import CommonAttentionMetadata, split_attn_metadata
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.worker.gpu_input_batch import CachedRequestState
from vllm.v1.worker.gpu_model_runner import GPUModelRunner, PerLayerAttnMetadata
@@ -81,10 +77,7 @@ class OmniGPUModelRunner(GPUModelRunner):
sampling_params = new_req_data.sampling_params
pooling_params = new_req_data.pooling_params
if (
sampling_params
and sampling_params.sampling_type == SamplingType.RANDOM_SEED
):
if sampling_params and sampling_params.sampling_type == SamplingType.RANDOM_SEED:
generator = torch.Generator(device=self.device)
generator.manual_seed(sampling_params.seed)
else:
@@ -145,9 +138,7 @@ class OmniGPUModelRunner(GPUModelRunner):
if isinstance(payload_info, AdditionalInformationPayload):
for k, entry in payload_info.entries.items():
if entry.tensor_data is not None:
dt = np.dtype(
getattr(entry, "tensor_dtype", "float32")
)
dt = np.dtype(getattr(entry, "tensor_dtype", "float32"))
arr = np.frombuffer(entry.tensor_data, dtype=dt)
arr = arr.reshape(entry.tensor_shape)
info_dict[k] = torch.from_numpy(arr)
@@ -188,9 +179,7 @@ class OmniGPUModelRunner(GPUModelRunner):
new_token_ids = req_data.new_token_ids[i]
# Add the sampled token(s) from the previous step (if any).
# This doesn't include "unverified" tokens like spec tokens.
num_new_tokens = (
num_computed_tokens + len(new_token_ids) - req_state.num_tokens
)
num_new_tokens = num_computed_tokens + len(new_token_ids) - req_state.num_tokens
if num_new_tokens == 1:
# Avoid slicing list in most common case.
req_state.output_token_ids.append(new_token_ids[-1])
@@ -228,23 +217,17 @@ class OmniGPUModelRunner(GPUModelRunner):
# Add new_token_ids to token_ids_cpu.
start_token_index = num_computed_tokens
end_token_index = num_computed_tokens + len(new_token_ids)
self.input_batch.token_ids_cpu[
req_index, start_token_index:end_token_index
] = new_token_ids
self.input_batch.token_ids_cpu[req_index, start_token_index:end_token_index] = new_token_ids
self.input_batch.num_tokens_no_spec[req_index] = end_token_index
self.input_batch.num_tokens[req_index] = end_token_index
# Add spec_token_ids to token_ids_cpu.
spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(
req_id, ()
)
spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id, ())
if spec_token_ids:
num_spec_tokens = len(spec_token_ids)
start_index = self.input_batch.num_tokens_no_spec[req_index]
end_token_index = start_index + num_spec_tokens
self.input_batch.token_ids_cpu[
req_index, start_index:end_token_index
] = spec_token_ids
self.input_batch.token_ids_cpu[req_index, start_index:end_token_index] = spec_token_ids
# NOTE(woosuk): `num_tokens` here may include spec tokens.
self.input_batch.num_tokens[req_index] += num_spec_tokens
@@ -261,20 +244,15 @@ class OmniGPUModelRunner(GPUModelRunner):
self.input_batch.refresh_metadata()
@torch.inference_mode()
def extract_multimodal_outputs(
self, hidden_states: Union[torch.Tensor, List[torch.Tensor]]
) -> dict:
if (
hasattr(self.model, "have_multimodal_outputs")
and self.model.have_multimodal_outputs
):
def extract_multimodal_outputs(self, hidden_states: Union[torch.Tensor, list[torch.Tensor]]) -> dict:
if hasattr(self.model, "have_multimodal_outputs") and self.model.have_multimodal_outputs:
text_hidden_states = hidden_states.text_hidden_states
multimodal_outputs = hidden_states.multimodal_outputs
elif isinstance(hidden_states, torch.Tensor):
text_hidden_states = hidden_states
multimodal_outputs = {}
elif isinstance(hidden_states, List):
elif isinstance(hidden_states, list):
text_hidden_states = hidden_states[0]
multimodal_outputs = {}
else:
@@ -427,37 +405,25 @@ class OmniGPUModelRunner(GPUModelRunner):
self.query_start_loc.np[1 : num_reqs + 1] = cum_num_tokens
self.query_start_loc.copy_to_gpu()
for kv_cache_group_id, kv_cache_group_spec in enumerate(
self.kv_cache_config.kv_cache_groups
):
for kv_cache_group_id, kv_cache_group_spec in enumerate(self.kv_cache_config.kv_cache_groups):
common_attn_metadata = CommonAttentionMetadata(
query_start_loc=self.query_start_loc.gpu[: num_reqs + 1],
query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs + 1],
seq_lens=self.seq_lens.gpu[:num_reqs],
seq_lens_cpu=self.seq_lens.cpu[:num_reqs],
num_computed_tokens_cpu=self.input_batch.num_computed_tokens_cpu_tensor[ # noqa: E501
:num_reqs
],
num_computed_tokens_cpu=self.input_batch.num_computed_tokens_cpu_tensor[:num_reqs], # noqa: E501
num_reqs=num_reqs,
num_actual_tokens=num_tokens,
max_query_len=max_query_len,
max_seq_len=self.max_model_len,
block_table_tensor=self.input_batch.block_table[
kv_cache_group_id
].get_device_tensor(num_reqs),
slot_mapping=self.input_batch.block_table[
kv_cache_group_id
].slot_mapping.gpu[:num_tokens],
block_table_tensor=self.input_batch.block_table[kv_cache_group_id].get_device_tensor(num_reqs),
slot_mapping=self.input_batch.block_table[kv_cache_group_id].slot_mapping.gpu[:num_tokens],
causal=True,
)
for attn_group in self.attn_groups[kv_cache_group_id]:
if ubatch_slices is not None:
common_attn_metadata_list = split_attn_metadata(
ubatch_slices, common_attn_metadata
)
for ubid, common_attn_metadata in enumerate(
common_attn_metadata_list
):
common_attn_metadata_list = split_attn_metadata(ubatch_slices, common_attn_metadata)
for ubid, common_attn_metadata in enumerate(common_attn_metadata_list):
assert common_attn_metadata.max_query_len == 1
attn_metadata_i = attn_group.get_metadata_builder(
ubatch_id=ubid
@@ -467,15 +433,13 @@ class OmniGPUModelRunner(GPUModelRunner):
attn_metadata[ubid][layer_name] = attn_metadata_i
else:
assert type(attn_metadata) is dict
attn_metadata_i = attn_group.get_metadata_builder().build_for_cudagraph_capture( # noqa: E501
attn_metadata_i = attn_group.get_metadata_builder().build_for_cudagraph_capture(
common_attn_metadata
)
) # noqa: E501
for layer_name in attn_group.layer_names:
attn_metadata[layer_name] = attn_metadata_i
with self.maybe_dummy_run_with_lora(
self.lora_config, num_scheduled_tokens, remove_lora
):
with self.maybe_dummy_run_with_lora(self.lora_config, num_scheduled_tokens, remove_lora):
model_kwargs = self._init_model_kwargs(num_tokens)
if self.supports_mm_inputs and not self.model_config.is_encoder_decoder:
input_ids = None
@@ -501,17 +465,13 @@ class OmniGPUModelRunner(GPUModelRunner):
intermediate_tensors = None
else:
if self.intermediate_tensors is None:
self.intermediate_tensors = (
self.model.make_empty_intermediate_tensors(
batch_size=self.max_num_tokens,
dtype=self.model_config.dtype,
device=self.device,
)
self.intermediate_tensors = self.model.make_empty_intermediate_tensors(
batch_size=self.max_num_tokens,
dtype=self.model_config.dtype,
device=self.device,
)
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
num_tokens, None, False
)
intermediate_tensors = self.sync_and_slice_intermediate_tensors(num_tokens, None, False)
# filter out the valid batch descriptor
_cg_mode, batch_descriptor = (
@@ -527,10 +487,7 @@ class OmniGPUModelRunner(GPUModelRunner):
if cudagraph_runtime_mode is not None:
# we allow forcing NONE when the dispatcher disagrees to support
# warm ups for cudagraph capture
assert (
cudagraph_runtime_mode == CUDAGraphMode.NONE
or cudagraph_runtime_mode == _cg_mode
), (
assert cudagraph_runtime_mode == CUDAGraphMode.NONE or cudagraph_runtime_mode == _cg_mode, (
f"Cudagraph runtime mode mismatch at dummy_run. "
f"Expected {_cg_mode}, but got {cudagraph_runtime_mode}."
)
@@ -585,7 +542,5 @@ class OmniGPUModelRunner(GPUModelRunner):
self.eplb_step(is_dummy=True, is_profile=is_profile)
logit_indices = np.cumsum(num_scheduled_tokens) - 1
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(
hidden_states
)
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states)
return hidden_states, hidden_states[logit_indices]