[core]refactor communication layer: PR1(Added Refactor Infra Only) (#1555)

Signed-off-by: natureofnature <wzliu@connect.hku.hk>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
NATUREandHongsheng Liu authored and GitHub committed 2026-04-13 16:01:14 +08:00
1 parent 2b70e89535
commit 0d4e975e1b
9 files changed
+4712 -3

No files matched your search

+2 -1
View File
@@ -40,6 +40,7 @@ from vllm.v1.worker.utils import is_residual_scattered_for_sp
from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager
from vllm_omni.outputs import OmniModelRunnerOutput
from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner
from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin
logger = init_logger(__name__)
@@ -60,7 +61,7 @@ class ExecuteModelState(NamedTuple):
slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None
class GPUARModelRunner(OmniGPUModelRunner):
class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
"""Autoregressive GPU model runner that returns hidden states per request.
Follows the v0.12 two-phase execute/sample flow from GPUModelRunner, and
@@ -39,11 +39,12 @@ from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
from vllm_omni.outputs import OmniModelRunnerOutput
from vllm_omni.worker.gpu_ar_model_runner import ExecuteModelState
from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner
from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin
logger = logging.getLogger(__name__)
class GPUGenerationModelRunner(OmniGPUModelRunner):
class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
"""Generation model runner for vLLM-Omni (non-autoregressive).
- Reuses GPUModelRunner preparation, multimodal handling, and TP/PP/DP glue.
File diff suppressed because it is too large. Load diff
+64
View File
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers for explicit thinker decode span metadata."""
from collections.abc import Mapping
from typing import Any
import torch
THINKER_DECODE_EMBEDDINGS_KEY = "thinker_decode_embeddings"
THINKER_OUTPUT_TOKEN_IDS_KEY = "thinker_output_token_ids"
THINKER_DECODE_TOKEN_START_KEY = "thinker_decode_embeddings_token_start"
THINKER_DECODE_TOKEN_END_KEY = "thinker_decode_embeddings_token_end"
CACHED_THINKER_DECODE_EMBEDDINGS_KEY = "cached_thinker_decode_embeddings"
CACHED_THINKER_DECODE_TOKEN_START_KEY = "cached_thinker_decode_embeddings_token_start"
CACHED_THINKER_DECODE_TOKEN_END_KEY = "cached_thinker_decode_embeddings_token_end"
TensorSpan = tuple[torch.Tensor, int, int]
def get_tensor_span(payload: Mapping[str, Any], *, tensor_key: str, start_key: str, end_key: str) -> TensorSpan | None:
tensor = payload.get(tensor_key)
start = payload.get(start_key)
end = payload.get(end_key)
if not isinstance(tensor, torch.Tensor):
return None
if not isinstance(start, int) or not isinstance(end, int):
return None
if start < 0 or end < start or (end - start) != int(tensor.shape[0]):
return None
return tensor, start, end
def merge_tensor_spans(existing_span: TensorSpan | None, incoming_span: TensorSpan | None) -> TensorSpan | None:
if existing_span is None or incoming_span is None:
return None
existing_tensor, existing_start, existing_end = existing_span
incoming_tensor, incoming_start, incoming_end = incoming_span
if incoming_tensor.device != existing_tensor.device or incoming_tensor.dtype != existing_tensor.dtype:
incoming_tensor = incoming_tensor.to(device=existing_tensor.device, dtype=existing_tensor.dtype)
if incoming_start == existing_end:
return torch.cat([existing_tensor, incoming_tensor], dim=0), existing_start, incoming_end
if incoming_start < existing_end:
overlap = existing_end - incoming_start
if overlap >= int(incoming_tensor.shape[0]):
return existing_tensor, existing_start, existing_end
trimmed_tensor = incoming_tensor[overlap:]
return (
torch.cat([existing_tensor, trimmed_tensor], dim=0),
existing_start,
existing_end + int(trimmed_tensor.shape[0]),
)
return None
def get_tensor_span_row(span: TensorSpan | None, index: int) -> torch.Tensor | None:
if span is None:
return None
tensor, start, end = span
if index < start or index >= end:
return None
return tensor[index - start]