[CI] fix ci and format existing code (#71)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
1 parent
09fab2ecc9
commit
5707fc78d5
52 files changed
+813
-2155
No files matched your search
@@ -1,8 +1,7 @@
|
||||
from typing import TYPE_CHECKING, List, Optional, Union, cast
|
||||
from typing import TYPE_CHECKING, Optional, Union, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
@@ -10,10 +9,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces_base import VllmModelForPooling
|
||||
from vllm.sampling_params import SamplingType
|
||||
from vllm.utils import LazyLoader, cdiv
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
CommonAttentionMetadata,
|
||||
split_attn_metadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import CommonAttentionMetadata, split_attn_metadata
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner, PerLayerAttnMetadata
|
||||
@@ -81,10 +77,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
sampling_params = new_req_data.sampling_params
|
||||
pooling_params = new_req_data.pooling_params
|
||||
|
||||
if (
|
||||
sampling_params
|
||||
and sampling_params.sampling_type == SamplingType.RANDOM_SEED
|
||||
):
|
||||
if sampling_params and sampling_params.sampling_type == SamplingType.RANDOM_SEED:
|
||||
generator = torch.Generator(device=self.device)
|
||||
generator.manual_seed(sampling_params.seed)
|
||||
else:
|
||||
@@ -145,9 +138,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if isinstance(payload_info, AdditionalInformationPayload):
|
||||
for k, entry in payload_info.entries.items():
|
||||
if entry.tensor_data is not None:
|
||||
dt = np.dtype(
|
||||
getattr(entry, "tensor_dtype", "float32")
|
||||
)
|
||||
dt = np.dtype(getattr(entry, "tensor_dtype", "float32"))
|
||||
arr = np.frombuffer(entry.tensor_data, dtype=dt)
|
||||
arr = arr.reshape(entry.tensor_shape)
|
||||
info_dict[k] = torch.from_numpy(arr)
|
||||
@@ -188,9 +179,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
new_token_ids = req_data.new_token_ids[i]
|
||||
# Add the sampled token(s) from the previous step (if any).
|
||||
# This doesn't include "unverified" tokens like spec tokens.
|
||||
num_new_tokens = (
|
||||
num_computed_tokens + len(new_token_ids) - req_state.num_tokens
|
||||
)
|
||||
num_new_tokens = num_computed_tokens + len(new_token_ids) - req_state.num_tokens
|
||||
if num_new_tokens == 1:
|
||||
# Avoid slicing list in most common case.
|
||||
req_state.output_token_ids.append(new_token_ids[-1])
|
||||
@@ -228,23 +217,17 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
# Add new_token_ids to token_ids_cpu.
|
||||
start_token_index = num_computed_tokens
|
||||
end_token_index = num_computed_tokens + len(new_token_ids)
|
||||
self.input_batch.token_ids_cpu[
|
||||
req_index, start_token_index:end_token_index
|
||||
] = new_token_ids
|
||||
self.input_batch.token_ids_cpu[req_index, start_token_index:end_token_index] = new_token_ids
|
||||
self.input_batch.num_tokens_no_spec[req_index] = end_token_index
|
||||
self.input_batch.num_tokens[req_index] = end_token_index
|
||||
|
||||
# Add spec_token_ids to token_ids_cpu.
|
||||
spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(
|
||||
req_id, ()
|
||||
)
|
||||
spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id, ())
|
||||
if spec_token_ids:
|
||||
num_spec_tokens = len(spec_token_ids)
|
||||
start_index = self.input_batch.num_tokens_no_spec[req_index]
|
||||
end_token_index = start_index + num_spec_tokens
|
||||
self.input_batch.token_ids_cpu[
|
||||
req_index, start_index:end_token_index
|
||||
] = spec_token_ids
|
||||
self.input_batch.token_ids_cpu[req_index, start_index:end_token_index] = spec_token_ids
|
||||
# NOTE(woosuk): `num_tokens` here may include spec tokens.
|
||||
self.input_batch.num_tokens[req_index] += num_spec_tokens
|
||||
|
||||
@@ -261,20 +244,15 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
self.input_batch.refresh_metadata()
|
||||
|
||||
@torch.inference_mode()
|
||||
def extract_multimodal_outputs(
|
||||
self, hidden_states: Union[torch.Tensor, List[torch.Tensor]]
|
||||
) -> dict:
|
||||
if (
|
||||
hasattr(self.model, "have_multimodal_outputs")
|
||||
and self.model.have_multimodal_outputs
|
||||
):
|
||||
def extract_multimodal_outputs(self, hidden_states: Union[torch.Tensor, list[torch.Tensor]]) -> dict:
|
||||
if hasattr(self.model, "have_multimodal_outputs") and self.model.have_multimodal_outputs:
|
||||
text_hidden_states = hidden_states.text_hidden_states
|
||||
multimodal_outputs = hidden_states.multimodal_outputs
|
||||
|
||||
elif isinstance(hidden_states, torch.Tensor):
|
||||
text_hidden_states = hidden_states
|
||||
multimodal_outputs = {}
|
||||
elif isinstance(hidden_states, List):
|
||||
elif isinstance(hidden_states, list):
|
||||
text_hidden_states = hidden_states[0]
|
||||
multimodal_outputs = {}
|
||||
else:
|
||||
@@ -427,37 +405,25 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
self.query_start_loc.np[1 : num_reqs + 1] = cum_num_tokens
|
||||
self.query_start_loc.copy_to_gpu()
|
||||
|
||||
for kv_cache_group_id, kv_cache_group_spec in enumerate(
|
||||
self.kv_cache_config.kv_cache_groups
|
||||
):
|
||||
for kv_cache_group_id, kv_cache_group_spec in enumerate(self.kv_cache_config.kv_cache_groups):
|
||||
common_attn_metadata = CommonAttentionMetadata(
|
||||
query_start_loc=self.query_start_loc.gpu[: num_reqs + 1],
|
||||
query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs + 1],
|
||||
seq_lens=self.seq_lens.gpu[:num_reqs],
|
||||
seq_lens_cpu=self.seq_lens.cpu[:num_reqs],
|
||||
num_computed_tokens_cpu=self.input_batch.num_computed_tokens_cpu_tensor[ # noqa: E501
|
||||
:num_reqs
|
||||
],
|
||||
num_computed_tokens_cpu=self.input_batch.num_computed_tokens_cpu_tensor[:num_reqs], # noqa: E501
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=num_tokens,
|
||||
max_query_len=max_query_len,
|
||||
max_seq_len=self.max_model_len,
|
||||
block_table_tensor=self.input_batch.block_table[
|
||||
kv_cache_group_id
|
||||
].get_device_tensor(num_reqs),
|
||||
slot_mapping=self.input_batch.block_table[
|
||||
kv_cache_group_id
|
||||
].slot_mapping.gpu[:num_tokens],
|
||||
block_table_tensor=self.input_batch.block_table[kv_cache_group_id].get_device_tensor(num_reqs),
|
||||
slot_mapping=self.input_batch.block_table[kv_cache_group_id].slot_mapping.gpu[:num_tokens],
|
||||
causal=True,
|
||||
)
|
||||
for attn_group in self.attn_groups[kv_cache_group_id]:
|
||||
if ubatch_slices is not None:
|
||||
common_attn_metadata_list = split_attn_metadata(
|
||||
ubatch_slices, common_attn_metadata
|
||||
)
|
||||
for ubid, common_attn_metadata in enumerate(
|
||||
common_attn_metadata_list
|
||||
):
|
||||
common_attn_metadata_list = split_attn_metadata(ubatch_slices, common_attn_metadata)
|
||||
for ubid, common_attn_metadata in enumerate(common_attn_metadata_list):
|
||||
assert common_attn_metadata.max_query_len == 1
|
||||
attn_metadata_i = attn_group.get_metadata_builder(
|
||||
ubatch_id=ubid
|
||||
@@ -467,15 +433,13 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
attn_metadata[ubid][layer_name] = attn_metadata_i
|
||||
else:
|
||||
assert type(attn_metadata) is dict
|
||||
attn_metadata_i = attn_group.get_metadata_builder().build_for_cudagraph_capture( # noqa: E501
|
||||
attn_metadata_i = attn_group.get_metadata_builder().build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
)
|
||||
) # noqa: E501
|
||||
for layer_name in attn_group.layer_names:
|
||||
attn_metadata[layer_name] = attn_metadata_i
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
self.lora_config, num_scheduled_tokens, remove_lora
|
||||
):
|
||||
with self.maybe_dummy_run_with_lora(self.lora_config, num_scheduled_tokens, remove_lora):
|
||||
model_kwargs = self._init_model_kwargs(num_tokens)
|
||||
if self.supports_mm_inputs and not self.model_config.is_encoder_decoder:
|
||||
input_ids = None
|
||||
@@ -501,17 +465,13 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
intermediate_tensors = None
|
||||
else:
|
||||
if self.intermediate_tensors is None:
|
||||
self.intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors(
|
||||
batch_size=self.max_num_tokens,
|
||||
dtype=self.model_config.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
self.intermediate_tensors = self.model.make_empty_intermediate_tensors(
|
||||
batch_size=self.max_num_tokens,
|
||||
dtype=self.model_config.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
|
||||
num_tokens, None, False
|
||||
)
|
||||
intermediate_tensors = self.sync_and_slice_intermediate_tensors(num_tokens, None, False)
|
||||
|
||||
# filter out the valid batch descriptor
|
||||
_cg_mode, batch_descriptor = (
|
||||
@@ -527,10 +487,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if cudagraph_runtime_mode is not None:
|
||||
# we allow forcing NONE when the dispatcher disagrees to support
|
||||
# warm ups for cudagraph capture
|
||||
assert (
|
||||
cudagraph_runtime_mode == CUDAGraphMode.NONE
|
||||
or cudagraph_runtime_mode == _cg_mode
|
||||
), (
|
||||
assert cudagraph_runtime_mode == CUDAGraphMode.NONE or cudagraph_runtime_mode == _cg_mode, (
|
||||
f"Cudagraph runtime mode mismatch at dummy_run. "
|
||||
f"Expected {_cg_mode}, but got {cudagraph_runtime_mode}."
|
||||
)
|
||||
@@ -585,7 +542,5 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
self.eplb_step(is_dummy=True, is_profile=is_profile)
|
||||
|
||||
logit_indices = np.cumsum(num_scheduled_tokens) - 1
|
||||
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(
|
||||
hidden_states
|
||||
)
|
||||
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states)
|
||||
return hidden_states, hidden_states[logit_indices]
|
||||
Reference in new issue
Block a user