[Rebase] Rebase to vllm 0.28.0 (#6606)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
da4a08b65b
commit
651b03225b
42 files changed
+931
-121
No files matched your search
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Code2Wav GPU Model Runner for vLLM-Omni.
|
||||
|
||||
Handles direct conversion from codec codes to audio waveforms for Qwen3 Omni MoE Code2Wav.
|
||||
@@ -192,9 +195,8 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
max_num_scheduled_tokens = int(num_scheduled_tokens_np.max())
|
||||
num_tokens_unpadded = scheduler_output.total_num_scheduled_tokens
|
||||
|
||||
logits_indices, spec_decode_metadata = self._prepare_inputs(
|
||||
scheduler_output,
|
||||
num_scheduled_tokens_np,
|
||||
logits_indices, spec_decode_metadata, max_num_sampled_tokens = self._prepare_inputs(
|
||||
scheduler_output, num_scheduled_tokens_np
|
||||
)
|
||||
|
||||
cascade_attn_prefix_lens = None
|
||||
@@ -278,6 +280,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
max_query_len=max_num_scheduled_tokens,
|
||||
ubatch_slices=ubatch_slices_attn,
|
||||
logits_indices=logits_indices,
|
||||
max_num_sampled_tokens=max_num_sampled_tokens,
|
||||
use_spec_decode=use_spec_decode,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
|
||||
@@ -309,14 +312,6 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
if getattr(self.model, "requires_request_ids", False):
|
||||
model_kwargs["request_ids"] = list(req_ids)
|
||||
|
||||
# Set cudagraph mode to none if calc_kv_scales is true.
|
||||
# KV scales calculation involves dynamic operations that are incompatible
|
||||
# with CUDA graph capture.
|
||||
if self.calculate_kv_scales:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
# Mark KV scales as calculated after the first forward pass
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
# Run the model.
|
||||
# Use persistent buffers for CUDA graphs.
|
||||
# When spec decode is enabled, defer connector finalization
|
||||
|
||||
Reference in new issue
Block a user