Use published release images and wire NPU auxiliary outputs
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
4ad71d8270
commit
4414701161
12 files changed
+294
-42
No files matched your search
@@ -1,9 +1,7 @@
|
||||
# Use the official release image with its matching CUDA/PyTorch stack.
|
||||
# The release images are available in upstream ECR before Docker Hub promotion.
|
||||
ARG VLLM_VERSION=0.31.0
|
||||
ARG VLLM_PRECOMPILED_WHEEL_COMMIT=db9527a46873454610df6dbedf79a36d6bf1a7f6
|
||||
ARG VLLM_BASE_IMAGE=public.ecr.aws/q9t5s3a7/vllm-release-repo
|
||||
ARG VLLM_BASE_TAG=${VLLM_PRECOMPILED_WHEEL_COMMIT}
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||
ARG VLLM_BASE_TAG=v${VLLM_VERSION}
|
||||
|
||||
# We use registry-based layer caching for the dependencies,
|
||||
# where the tag is determined based on the hash of the contents
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
# Official vLLM v0.31.0 release image, before Docker Hub tag promotion.
|
||||
ARG BASE_IMAGE=public.ecr.aws/q9t5s3a7/vllm-release-repo:db9527a46873454610df6dbedf79a36d6bf1a7f6
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.31.0
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG COMMON_WORKDIR=/app
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# Keep the ROCm runtime aligned with the vLLM release targeted by
|
||||
# docker/Dockerfile.ci. A stale base image can build successfully while failing
|
||||
# at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
|
||||
# Official vLLM v0.31.0 release image, before Docker Hub tag promotion.
|
||||
ARG BASE_IMAGE=public.ecr.aws/q9t5s3a7/vllm-release-repo:db9527a46873454610df6dbedf79a36d6bf1a7f6-rocm
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.31.0
|
||||
|
||||
# Keep the dependency-heavy portion independent from the source tree. The
|
||||
# registry cache is keyed on these inputs, while the final source copy can be
|
||||
|
||||
@@ -38,9 +38,11 @@ def test_rocm_base_tracks_cuda_vllm_release() -> None:
|
||||
|
||||
image_ref, separator, image_tag = rocm_base.rpartition(":")
|
||||
assert separator, f"expected a tagged ROCm base image, got {rocm_base}"
|
||||
assert image_ref == cuda_image
|
||||
assert image_tag == f"{cuda_release}-rocm"
|
||||
assert cuda_release == _docker_arg(CUDA_CI_DOCKERFILE, "VLLM_PRECOMPILED_WHEEL_COMMIT")
|
||||
assert cuda_image == "vllm/vllm-openai"
|
||||
assert image_ref == "vllm/vllm-openai-rocm"
|
||||
assert image_tag == cuda_release == f"v{_docker_arg(CUDA_CI_DOCKERFILE, 'VLLM_VERSION')}"
|
||||
assert _docker_arg(CUDA_CI_DOCKERFILE, "VLLM_BASE_IMAGE") == cuda_image
|
||||
assert _docker_arg(CUDA_CI_DOCKERFILE, "VLLM_BASE_TAG") == "v${VLLM_VERSION}"
|
||||
|
||||
|
||||
def test_rocm_defaults_to_prebuilt_base_image() -> None:
|
||||
|
||||
@@ -61,12 +61,14 @@ def test_internal_blacklist_keys_derived_from_orchestrator():
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
|
||||
def test_aux_outputs_require_the_actual_omni_stage_v2_runner(aux_args, use_v2_model_runner, monkeypatch):
|
||||
@pytest.mark.parametrize("is_npu", [False, True])
|
||||
def test_aux_outputs_require_a_supported_omni_stage_runner(aux_args, use_v2_model_runner, is_npu, monkeypatch):
|
||||
# Upstream runner selection reads this environment variable; Omni workers
|
||||
# select their runner from the separate per-stage flag instead.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
if use_v2_model_runner:
|
||||
args = OmniEngineArgs(model="unused", use_v2_model_runner=True, **aux_args)
|
||||
monkeypatch.setattr("vllm_omni.engine.arg_utils.current_omni_platform.is_npu", lambda: is_npu)
|
||||
if use_v2_model_runner or is_npu:
|
||||
args = OmniEngineArgs(model="unused", use_v2_model_runner=use_v2_model_runner, **aux_args)
|
||||
assert args.aux_output_config.enabled
|
||||
else:
|
||||
with pytest.raises(ValueError, match="use_v2_model_runner=True for this Omni stage"):
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU contract coverage for NPU routing capture and the native data plane."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.config import AuxOutputConfig
|
||||
from vllm.distributed.aux_output_connector import worker
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxOutputConnectorMetadata,
|
||||
AuxOutputSchedulerConnector,
|
||||
PackedBlockHashes,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import routed_experts_capturer
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
from vllm_omni.platforms.npu.worker.aux_output import NPUAuxOutputMixin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _Router:
|
||||
def _select_experts(self, logical_ids):
|
||||
return torch.ones_like(logical_ids, dtype=torch.float32), logical_ids
|
||||
|
||||
|
||||
class _AscendMoE(torch.nn.Module):
|
||||
__module__ = "vllm_ascend.ops.fused_moe"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.layer_id = 0
|
||||
self.router = _Router()
|
||||
|
||||
|
||||
class _Runner(NPUAuxOutputMixin):
|
||||
def _sync_device(self):
|
||||
self.synchronized += 1
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runner(monkeypatch):
|
||||
# Exercise the real capturer, block store, worker and scheduler connectors
|
||||
# with CPU tensors; importing vllm-ascend or owning NPU hardware is unnecessary.
|
||||
monkeypatch.setattr(routed_experts_capturer, "current_platform", SimpleNamespace(device_type="cpu"))
|
||||
monkeypatch.setattr(routed_experts_capturer, "get_forward_context", lambda: SimpleNamespace(dp_metadata=None))
|
||||
monkeypatch.setattr(worker, "get_tp_group", lambda: SimpleNamespace(is_first_rank=True))
|
||||
runner = _Runner()
|
||||
runner.model = torch.nn.Sequential(_AscendMoE())
|
||||
runner.vllm_config = SimpleNamespace(
|
||||
aux_output_config=AuxOutputConfig(enable_return_routed_experts=True),
|
||||
model_config=SimpleNamespace(
|
||||
get_total_num_hidden_layers=lambda: 1,
|
||||
get_num_experts=lambda: 16,
|
||||
get_num_experts_per_tok=lambda: 2,
|
||||
hf_text_config=SimpleNamespace(model_type="test_moe"),
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=16, max_num_seqs=4),
|
||||
parallel_config=SimpleNamespace(data_parallel_rank=0, tensor_parallel_size=1, decode_context_parallel_size=1),
|
||||
cache_config=SimpleNamespace(block_size=4, prefix_match_unit=None),
|
||||
max_concurrent_batches=2,
|
||||
)
|
||||
runner.input_batch = SimpleNamespace(req_ids=["r"], num_computed_tokens_cpu=np.array([0]))
|
||||
runner.query_start_loc = SimpleNamespace(np=np.array([0, 4]))
|
||||
runner.synchronized = 0
|
||||
runner._init_omni_aux_output(KVCacheConfig(8, [], []))
|
||||
try:
|
||||
yield runner
|
||||
finally:
|
||||
runner._close_omni_aux_output()
|
||||
|
||||
|
||||
def _step(runner, *, emit_start=0, hashes=None, finished=()):
|
||||
scheduler_output = SimpleNamespace(
|
||||
aux_output_connector_metadata=AuxOutputConnectorMetadata(
|
||||
generation=0,
|
||||
requests={} if finished else {"r": emit_start},
|
||||
block_hashes=hashes or {},
|
||||
finished_requests=finished,
|
||||
),
|
||||
scheduled_spec_decode_tokens={},
|
||||
)
|
||||
runner._begin_omni_aux_output_step(scheduler_output)
|
||||
return scheduler_output
|
||||
|
||||
|
||||
def _capture(runner, ids):
|
||||
logical_ids = torch.tensor(ids, dtype=torch.int32)
|
||||
weights, returned = runner.model[0].router._select_experts(logical_ids)
|
||||
assert torch.equal(returned, logical_ids)
|
||||
assert weights.shape == logical_ids.shape
|
||||
return logical_ids.numpy()[:, None, :].astype(np.uint8)
|
||||
|
||||
|
||||
def test_routing_reaches_scheduler_and_owns_legacy_batch_snapshot(runner):
|
||||
step = _step(runner)
|
||||
expected = _capture(runner, [[0, 1], [1, 2], [2, 3], [3, 4]])
|
||||
pending = runner._prepare_omni_aux_output()
|
||||
# Simulate bookkeeping and the next batch reusing the runner's buffers.
|
||||
runner.input_batch.num_computed_tokens_cpu[:] = 99
|
||||
runner.query_start_loc.np[:] = 99
|
||||
_capture(runner, [[7, 8]] * 4)
|
||||
output = runner._finish_omni_aux_output(pending, step, torch.tensor([[5]]))
|
||||
request = SimpleNamespace(request_id="r", num_tokens=5)
|
||||
rows = AuxOutputSchedulerConnector().take_output(request, output)
|
||||
np.testing.assert_array_equal(rows, expected)
|
||||
assert output["r"].token_start == 0
|
||||
assert runner.synchronized == 1
|
||||
|
||||
|
||||
def test_speculative_rejection_drops_unaccepted_routing_suffix(runner):
|
||||
step = _step(runner)
|
||||
step.scheduled_spec_decode_tokens = {"r": [7, 8, 9]}
|
||||
expected = _capture(runner, [[0, 1], [1, 2], [2, 3], [3, 4]])
|
||||
pending = runner._prepare_omni_aux_output()
|
||||
output = runner._finish_omni_aux_output(pending, step, torch.tensor([[5, 6, -1, -1]]))
|
||||
np.testing.assert_array_equal(output["r"].rows, expected[:2])
|
||||
|
||||
|
||||
def test_partial_prefill_accumulates_rows_and_no_forward_step_tears_down(runner):
|
||||
step = _step(runner)
|
||||
prefix = _capture(runner, [[0, 1], [1, 2], [2, 3], [3, 4]])
|
||||
pending = runner._prepare_omni_aux_output()
|
||||
assert runner._finish_omni_aux_output(pending, step, torch.tensor([[5]]), [0]) == {}
|
||||
|
||||
step = _step(runner, hashes={"r": PackedBlockHashes(b"h0", 2)})
|
||||
runner.input_batch.num_computed_tokens_cpu[:] = 4
|
||||
suffix = _capture(runner, [[4, 5], [5, 6], [6, 7], [7, 8]])
|
||||
pending = runner._prepare_omni_aux_output()
|
||||
output = runner._finish_omni_aux_output(pending, step, torch.tensor([[5]]))
|
||||
np.testing.assert_array_equal(output["r"].rows, np.concatenate((prefix, suffix)))
|
||||
_step(runner, finished=("r",))
|
||||
assert not runner.aux_output_connector._requests
|
||||
store = runner.aux_output_connector._store
|
||||
runner._close_omni_aux_output()
|
||||
assert not store._thread.is_alive()
|
||||
assert runner.model[0].capture_fn is None
|
||||
runner._close_omni_aux_output() # shutdown is idempotent
|
||||
|
||||
|
||||
def test_reinitialization_does_not_wrap_router_twice(runner):
|
||||
selection = runner.model[0].router._select_experts
|
||||
runner._init_omni_aux_output(KVCacheConfig(8, [], []))
|
||||
assert runner.model[0].router._select_experts is selection
|
||||
step = _step(runner)
|
||||
expected = _capture(runner, [[0, 1]] * 4)
|
||||
output = runner._finish_omni_aux_output(runner._prepare_omni_aux_output(), step, torch.tensor([[5]]))
|
||||
np.testing.assert_array_equal(output["r"].rows, expected)
|
||||
|
||||
|
||||
def test_disabled_capture_preserves_normal_runner_path(runner):
|
||||
runner._close_omni_aux_output()
|
||||
runner.vllm_config.aux_output_config.enable_return_routed_experts = False
|
||||
runner._init_omni_aux_output(KVCacheConfig(8, [], []))
|
||||
runner._begin_omni_aux_output_step(SimpleNamespace())
|
||||
assert runner._prepare_omni_aux_output() is None
|
||||
assert runner._finish_omni_aux_output(None, SimpleNamespace()) is None
|
||||
@@ -270,7 +270,9 @@ class OmniEngineArgs(EngineArgs):
|
||||
)
|
||||
validate_worker_omni_connector(self.worker_cls, needs_connector)
|
||||
super().__post_init__()
|
||||
if self.aux_output_config.enabled and not self.use_v2_model_runner:
|
||||
# The NPU runner implements the auxiliary connector on its legacy
|
||||
# execution path; CUDA/ROCm Omni stages use the V2 implementation.
|
||||
if self.aux_output_config.enabled and not self.use_v2_model_runner and not current_omni_platform.is_npu():
|
||||
raise ValueError("Auxiliary outputs require use_v2_model_runner=True for this Omni stage.")
|
||||
|
||||
def _ensure_omni_models_registered(self):
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Auxiliary-output delivery for Ascend's legacy model runner."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _install_ascend_capture_sources(model: Any) -> None:
|
||||
# AscendRoutedExperts invokes the router's private selection method, before
|
||||
# logical-to-physical remapping and shared-expert expansion. It bypasses
|
||||
# BaseRouter.select_experts(), where upstream normally invokes capture_fn.
|
||||
for module in model.modules():
|
||||
if not type(module).__module__.startswith("vllm_ascend.") or not hasattr(module, "layer_id"):
|
||||
continue
|
||||
router = getattr(module, "router", None)
|
||||
select_experts = getattr(router, "_select_experts", None)
|
||||
if not callable(select_experts) or getattr(module, "_omni_npu_capture_source", False):
|
||||
continue
|
||||
module.capture_fn = None
|
||||
|
||||
def select_and_capture(*args, _select=select_experts, _source=module, **kwargs):
|
||||
weights, logical_ids = _select(*args, **kwargs)
|
||||
if _source.capture_fn is not None:
|
||||
_source.capture_fn(logical_ids)
|
||||
return weights, logical_ids
|
||||
|
||||
router._select_experts = select_and_capture
|
||||
# layer_id + capture_fn implement upstream RoutedExpertsCaptureSource.
|
||||
module._omni_npu_capture_source = True
|
||||
|
||||
|
||||
class NPUAuxOutputMixin:
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.aux_output_connector = None
|
||||
|
||||
def _init_omni_aux_output(self, kv_cache_config) -> None:
|
||||
if not self.vllm_config.aux_output_config.enabled:
|
||||
return
|
||||
from vllm.distributed.aux_output_connector.worker import get_aux_output_connector
|
||||
|
||||
self._close_omni_aux_output()
|
||||
_install_ascend_capture_sources(self.model)
|
||||
self.aux_output_connector = get_aux_output_connector(self.model, self.vllm_config, kv_cache_config)
|
||||
|
||||
def _begin_omni_aux_output_step(self, scheduler_output) -> None:
|
||||
if self.aux_output_connector is not None:
|
||||
# Apply terminal/hash-only metadata even on no-forward steps.
|
||||
self.aux_output_connector.begin_step(scheduler_output.aux_output_connector_metadata)
|
||||
|
||||
def _prepare_omni_aux_output(self):
|
||||
if self.aux_output_connector is None:
|
||||
return None
|
||||
num_reqs = len(self.input_batch.req_ids)
|
||||
# Bookkeeping and the next async batch mutate these legacy buffers.
|
||||
batch = SimpleNamespace(
|
||||
req_ids=list(self.input_batch.req_ids),
|
||||
query_start_loc_np=self.query_start_loc.np[: num_reqs + 1].copy(),
|
||||
num_computed_tokens_np=self.input_batch.num_computed_tokens_cpu[:num_reqs].copy(),
|
||||
)
|
||||
return self.aux_output_connector.prepare_output(batch)
|
||||
|
||||
def _finish_omni_aux_output(self, pending, scheduler_output, sampled_token_ids=None, invalid_req_indices=()):
|
||||
if pending is None:
|
||||
return None
|
||||
num_reqs = len(pending.request_ids)
|
||||
num_sampled = np.zeros(num_reqs, dtype=np.int32)
|
||||
num_rejected = np.zeros(num_reqs, dtype=np.int32)
|
||||
if sampled_token_ids is not None:
|
||||
# Instrumentation opts into a synchronized copy. The normal NPU
|
||||
# async sampling path remains unchanged when capture is disabled.
|
||||
sampled = sampled_token_ids.detach().to("cpu").numpy()
|
||||
num_sampled[:] = (sampled >= 0).sum(axis=-1)
|
||||
for i, request_id in enumerate(pending.request_ids):
|
||||
draft_count = len(scheduler_output.scheduled_spec_decode_tokens.get(request_id, ()))
|
||||
if draft_count:
|
||||
num_rejected[i] = max(0, draft_count + 1 - int(num_sampled[i]))
|
||||
# Partial prefills execute valid routing rows but emit no token.
|
||||
num_sampled[list(invalid_req_indices)] = 0
|
||||
pending.enqueue_cpu_copy(num_sampled=num_sampled, num_rejected=num_rejected)
|
||||
self._sync_device()
|
||||
return pending.process_output()
|
||||
|
||||
def _close_omni_aux_output(self) -> None:
|
||||
connector = self.aux_output_connector
|
||||
self.aux_output_connector = None
|
||||
if connector is not None:
|
||||
try:
|
||||
connector.close()
|
||||
finally:
|
||||
for module in self.model.modules():
|
||||
if getattr(module, "_omni_npu_capture_source", False):
|
||||
module.capture_fn = None
|
||||
@@ -59,3 +59,11 @@ class OmniNPUWorkerBase(NPUWorker):
|
||||
self.profiler.start()
|
||||
else:
|
||||
self.profiler.stop()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
try:
|
||||
runner = getattr(self, "model_runner", None)
|
||||
if runner is not None:
|
||||
runner._close_omni_aux_output()
|
||||
finally:
|
||||
super().shutdown()
|
||||
@@ -315,10 +315,6 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
scheduler_output: SchedulerOutput,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
) -> OmniModelRunnerOutput | IntermediateTensors | None:
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
capturer = self.routed_experts_capturer
|
||||
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
|
||||
capturer.finalize_pending_copy()
|
||||
profiling_chunk_config = self.ascend_config.scheduler_config.profiling_chunk_config
|
||||
if profiling_chunk_config.enabled and profiling_chunk_config.need_timing:
|
||||
if getattr(scheduler_output, "disable_profiling_timing", False):
|
||||
@@ -392,6 +388,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
get_kv_transfer_group().handle_preemptions(kv_connector_metadata)
|
||||
# -------------------------------------- Omni-new -------------------------------------------------
|
||||
|
||||
self._begin_omni_aux_output_step(scheduler_output)
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
with record_function_or_nullcontext("prepare input"):
|
||||
with self.synchronize_input_prep():
|
||||
@@ -791,9 +788,6 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
|
||||
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(self, "_positions_cpu"):
|
||||
self._omni_routed_experts_d2h(scheduler_output)
|
||||
|
||||
return None
|
||||
|
||||
def _sample(
|
||||
@@ -884,6 +878,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
multimodal_outputs, # Omni-Specific
|
||||
prefix_cache_step_id,
|
||||
) = self.execute_model_state
|
||||
pending_aux_output = self._prepare_omni_aux_output()
|
||||
# Clear ephemeral state.
|
||||
self.execute_model_state = None
|
||||
hidden_seq_len = int(hidden_states.shape[0])
|
||||
@@ -1015,11 +1010,9 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
draft_by_req_id = dict(zip(draft_req_ids, draft_ids_list))
|
||||
output_spec_token_ids = [draft_by_req_id.get(req_id, []) for req_id in req_ids_output_copy]
|
||||
|
||||
routed_experts_lists = None
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
capturer = self.routed_experts_capturer
|
||||
if capturer is not None and hasattr(self.input_batch, "num_tokens_no_spec"):
|
||||
routed_experts_lists = self._omni_extract_routed_experts(scheduler_output)
|
||||
aux_output = self._finish_omni_aux_output(
|
||||
pending_aux_output, scheduler_output, sampler_output.sampled_token_ids, invalid_req_indices
|
||||
)
|
||||
|
||||
# -------------------------------------- Omni-new -------------------------------------------------
|
||||
engine_output_type, downstream_req_ids = self._resolve_pooler_payload_req_ids(req_ids_output_copy)
|
||||
@@ -1209,7 +1202,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
num_nans_in_logits=num_nans_in_logits,
|
||||
)
|
||||
model_runner_output.kv_extracted_req_ids = kv_extracted_req_ids
|
||||
model_runner_output.routed_experts = routed_experts_lists
|
||||
model_runner_output.aux_output_connector_output = aux_output
|
||||
model_runner_output.spec_token_ids = output_spec_token_ids
|
||||
with record_function_or_nullcontext("omni_output_builder:get_omni_connector_output"):
|
||||
model_runner_output.omni_connector_output = self.get_omni_connector_output()
|
||||
|
||||
@@ -89,10 +89,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
scheduler_output: SchedulerOutput,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
) -> OmniModelRunnerOutput | IntermediateTensors | None:
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
capturer = self.routed_experts_capturer
|
||||
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
|
||||
capturer.finalize_pending_copy()
|
||||
profiling_chunk_config = self.ascend_config.scheduler_config.profiling_chunk_config
|
||||
if profiling_chunk_config.enabled and profiling_chunk_config.need_timing:
|
||||
if getattr(scheduler_output, "disable_profiling_timing", False):
|
||||
@@ -127,6 +123,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
if flush_ids:
|
||||
self.flush_full_payload_outputs(flush_ids)
|
||||
|
||||
self._begin_omni_aux_output_step(scheduler_output)
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
with record_function_or_nullcontext("prepare input"):
|
||||
# -------------------------------------- Omni-new -------------------------------------------------
|
||||
@@ -418,9 +415,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
|
||||
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(self, "_positions_cpu"):
|
||||
self._omni_routed_experts_d2h(scheduler_output)
|
||||
|
||||
return None
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -461,6 +455,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
multimodal_outputs_raw, # Omni-Specific
|
||||
_prefix_cache_step_id, # generation stages never save to the prefix cache
|
||||
) = self.execute_model_state
|
||||
pending_aux_output = self._prepare_omni_aux_output()
|
||||
# Clear ephemeral state.
|
||||
self.execute_model_state = None
|
||||
|
||||
@@ -518,11 +513,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
req_state = self.requests.get(rid)
|
||||
if req_state is not None and inter_stage_outputs[i]:
|
||||
self.accumulate_full_payload_output(rid, inter_stage_outputs[i], req_state)
|
||||
routed_experts_lists = None
|
||||
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(
|
||||
self.input_batch, "num_tokens_no_spec"
|
||||
):
|
||||
routed_experts_lists = self._omni_extract_routed_experts(scheduler_output)
|
||||
aux_output = self._finish_omni_aux_output(pending_aux_output, scheduler_output)
|
||||
output = OmniModelRunnerOutput(
|
||||
req_ids=req_ids_output_copy,
|
||||
req_id_to_index=req_id_to_index_output_copy,
|
||||
@@ -537,7 +528,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
|
||||
)
|
||||
output.routed_experts = routed_experts_lists
|
||||
output.aux_output_connector_output = aux_output
|
||||
output.omni_connector_output = self.get_omni_connector_output()
|
||||
# -------------------------------------- Omni-new -------------------------------------------------
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm_ascend.worker.model_runner_v1 import SEQ_LEN_WITH_MAX_PA_WORKSPACE
|
||||
from vllm_omni.core.prefix_cache import stage_prefix_cache_config
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.platforms.npu._310p import is_310p
|
||||
from vllm_omni.platforms.npu.worker.aux_output import NPUAuxOutputMixin
|
||||
from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -35,7 +36,7 @@ else:
|
||||
from vllm_ascend.worker.model_runner_v1 import NPUModelRunner
|
||||
|
||||
|
||||
class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
|
||||
class OmniNPUModelRunner(NPUAuxOutputMixin, OmniGPUModelRunner, NPUModelRunner):
|
||||
def initialize_kv_cache(
|
||||
self, kv_cache_config, kv_cache_allocation_context: AbstractContextManager | None = None
|
||||
) -> None:
|
||||
@@ -43,6 +44,7 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
|
||||
NPUModelRunner.initialize_kv_cache(
|
||||
self, kv_cache_config, kv_cache_allocation_context=kv_cache_allocation_context
|
||||
)
|
||||
self._init_omni_aux_output(kv_cache_config)
|
||||
if getattr(self, "_omni_prefix_cache_cfg", None) is None:
|
||||
# Same gate as the GPU runner (pooling stage, kv_consumer /
|
||||
# kv_both, hybrid kv groups). Read the config back off
|
||||
|
||||
Reference in new issue
Block a user