[Rebase] Rebase to vllm 0.28.0 (#6606)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
da4a08b65b
commit
651b03225b
42 files changed
+931
-121
No files matched your search
@@ -100,6 +100,7 @@ def _make_drain_sched(session: Request) -> MagicMock:
|
||||
sched.pending_stop_after_extraction = set()
|
||||
sched.connector = None
|
||||
sched.kv_cache_manager.take_events.return_value = None
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.finished_req_ids_dict = {}
|
||||
sched.make_stats.return_value = None
|
||||
return sched
|
||||
|
||||
@@ -93,6 +93,7 @@ def _run_resumable_segment_stop(
|
||||
sched.pending_stop_after_extraction = set()
|
||||
sched.connector = None
|
||||
sched.kv_cache_manager.take_events.return_value = None
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.finished_req_ids_dict = {}
|
||||
sched.make_stats.return_value = None
|
||||
|
||||
@@ -227,6 +228,7 @@ def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> Non
|
||||
sched.pending_stop_after_extraction = set()
|
||||
sched.connector = None
|
||||
sched.kv_cache_manager.take_events.return_value = None
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.finished_req_ids_dict = {}
|
||||
sched.make_stats.return_value = None
|
||||
|
||||
|
||||
@@ -138,6 +138,7 @@ def test_resumable_generation_stop_marks_segment_boundary() -> None:
|
||||
sched.recompute_kv_load_failures = False
|
||||
sched.connector = None
|
||||
sched.kv_cache_manager.take_events.return_value = None
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.finished_req_ids_dict = {}
|
||||
sched.make_stats.return_value = None
|
||||
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""v0.28 deferred KV-block free fence in the Omni schedulers (#6606 review).
|
||||
|
||||
Upstream v0.28 (defer_block_free) advances ``sched_step_seq`` for every
|
||||
non-empty scheduled step and, at the top of ``update_from_output()``,
|
||||
advances ``processed_step_seq`` and drains the deferred frees. The Omni
|
||||
schedulers each carried only half of that contract:
|
||||
|
||||
- ``OmniARScheduler.schedule()`` delegates to ``super().schedule()`` (fence
|
||||
advances) but its reimplemented ``update_from_output()`` never advanced
|
||||
``processed_step_seq`` nor drained -> deferred blocks accumulate without
|
||||
bound under async/PP KV-consumer configurations.
|
||||
- ``OmniGenerationScheduler`` reimplements both: the fast-path
|
||||
``schedule()`` never advanced ``sched_step_seq`` (so finished requests
|
||||
freed blocks an in-flight GPU step could still be writing) and its
|
||||
``update_from_output()`` never drained.
|
||||
|
||||
These tests pin the restored contract at the call boundary; the drain and
|
||||
free logic itself is upstream-tested.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
|
||||
# before Request / RequestStatus are bound in this module.
|
||||
# isort: off
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.request_queue import SchedulingPolicy
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
|
||||
# isort: on
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_update_sched(scheduler_cls) -> MagicMock:
|
||||
"""MagicMock scheduler driving the real ``update_from_output`` with an
|
||||
empty batch; only the fence state carries real values."""
|
||||
sched = MagicMock()
|
||||
sched.requests = {}
|
||||
sched.perf_metrics = None
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.defer_block_free = True
|
||||
sched.processed_step_seq = 0
|
||||
return sched
|
||||
|
||||
|
||||
def _empty_frame(*, total_num_scheduled_tokens: int):
|
||||
scheduler_output = MagicMock(spec=SchedulerOutput)
|
||||
scheduler_output.num_scheduled_tokens = {}
|
||||
scheduler_output.total_num_scheduled_tokens = total_num_scheduled_tokens
|
||||
model_runner_output = MagicMock(spec=ModelRunnerOutput)
|
||||
model_runner_output.sampled_token_ids = []
|
||||
model_runner_output.logprobs = None
|
||||
model_runner_output.prompt_logprobs_dict = {}
|
||||
model_runner_output.pooler_output = None
|
||||
model_runner_output.num_nans_in_logits = None
|
||||
model_runner_output.kv_connector_output = None
|
||||
model_runner_output.cudagraph_stats = None
|
||||
return scheduler_output, model_runner_output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
|
||||
def test_update_from_output_advances_fence_and_drains(scheduler_cls) -> None:
|
||||
sched = _make_update_sched(scheduler_cls)
|
||||
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=3)
|
||||
|
||||
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
|
||||
|
||||
assert sched.processed_step_seq == 1
|
||||
sched._drain_deferred_frees.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
|
||||
def test_update_from_output_zero_token_step_leaves_fence(scheduler_cls) -> None:
|
||||
"""0-token steps do not advance the fence upstream (their seq was never
|
||||
advanced in schedule()); draining on them would run ahead of the GPU."""
|
||||
sched = _make_update_sched(scheduler_cls)
|
||||
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=0)
|
||||
|
||||
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
|
||||
|
||||
assert sched.processed_step_seq == 0
|
||||
sched._drain_deferred_frees.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheduler_cls", [OmniARScheduler, OmniGenerationScheduler])
|
||||
def test_update_from_output_defer_disabled_leaves_fence(scheduler_cls) -> None:
|
||||
sched = _make_update_sched(scheduler_cls)
|
||||
sched.defer_block_free = False
|
||||
scheduler_output, model_runner_output = _empty_frame(total_num_scheduled_tokens=3)
|
||||
|
||||
scheduler_cls.update_from_output(sched, scheduler_output, model_runner_output)
|
||||
|
||||
assert sched.processed_step_seq == 0
|
||||
sched._drain_deferred_frees.assert_not_called()
|
||||
|
||||
|
||||
def _make_fast_path_sched(*, defer_block_free: bool) -> tuple[MagicMock, Request]:
|
||||
"""MagicMock scheduler that drives the real generation fast-path
|
||||
``schedule()`` over one running one-shot request."""
|
||||
request = Request(
|
||||
request_id="req-fence",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=4),
|
||||
pooling_params=None,
|
||||
arrival_time=100.0,
|
||||
block_hasher=None,
|
||||
)
|
||||
request.status = RequestStatus.RUNNING
|
||||
|
||||
sched = MagicMock()
|
||||
sched.max_num_scheduled_tokens = 8
|
||||
sched._pause_state = PauseState.UNPAUSED
|
||||
sched.requests = {request.request_id: request}
|
||||
sched.running = [request]
|
||||
sched.waiting = [] # falsy: the waiting loop never runs
|
||||
sched.policy = SchedulingPolicy.FCFS
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.scheduler_config = MagicMock(enable_chunked_prefill=True)
|
||||
sched.num_lookahead_tokens = 0
|
||||
sched.log_stats = False
|
||||
sched.kv_cache_config = MagicMock(kv_cache_groups=[])
|
||||
sched.use_v2_model_runner = False
|
||||
sched.prev_step_scheduled_req_ids = set()
|
||||
sched.needs_kv_cache_zeroing = False
|
||||
sched.connector = None
|
||||
sched.ec_connector = None
|
||||
sched.defer_block_free = defer_block_free
|
||||
sched.sched_step_seq = 0
|
||||
# last_sched_seq stamping order: _update_after_schedule must observe the
|
||||
# already-advanced fence value, as upstream schedule() guarantees.
|
||||
sched.seq_at_update_after_schedule = None
|
||||
|
||||
def _capture(_scheduler_output):
|
||||
sched.seq_at_update_after_schedule = sched.sched_step_seq
|
||||
|
||||
sched._update_after_schedule.side_effect = _capture
|
||||
return sched, request
|
||||
|
||||
|
||||
def test_fast_path_schedule_advances_fence_before_update_after_schedule() -> None:
|
||||
sched, request = _make_fast_path_sched(defer_block_free=True)
|
||||
|
||||
OmniGenerationScheduler.schedule(sched)
|
||||
|
||||
assert sched.sched_step_seq == 1
|
||||
assert sched.seq_at_update_after_schedule == 1
|
||||
|
||||
|
||||
def test_fast_path_schedule_defer_disabled_leaves_fence() -> None:
|
||||
sched, request = _make_fast_path_sched(defer_block_free=False)
|
||||
|
||||
OmniGenerationScheduler.schedule(sched)
|
||||
|
||||
assert sched.sched_step_seq == 0
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""v0.28 EC request-finish contract in the Omni schedulers (#6606 review).
|
||||
|
||||
Upstream v0.28 ``_free_request()`` fires ``ec_connector.request_finished()``
|
||||
BEFORE the encoder cache is freed (so the connector can inspect per-request
|
||||
state), honors its delayed-free result, and returns the generated
|
||||
``ec_transfer_params``; ``update_from_output()`` then carries them into the
|
||||
``EngineCoreOutput``. ``OmniARScheduler`` previously skipped the hook and
|
||||
always returned ``None``; both Omni schedulers discarded the second tuple
|
||||
item, so a remote encoder-cache handle could never reach the frontend.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
|
||||
# before Request / RequestStatus are bound in this module.
|
||||
# isort: off
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
|
||||
# isort: on
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
EC_PARAMS = {"remote_encoder_cache_handle": "ec-1"}
|
||||
|
||||
|
||||
def _make_free_request_sched(*, ec_connector) -> tuple[OmniARScheduler, MagicMock]:
|
||||
"""Minimal real-`_free_request` harness; `order` records the EC hook
|
||||
firing relative to the encoder-cache free."""
|
||||
order = MagicMock()
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched._omits_kv_transfer_cache = {}
|
||||
sched._connector_finished = lambda request: (False, None)
|
||||
sched.ec_connector = ec_connector
|
||||
if ec_connector is not None:
|
||||
order.attach_mock(ec_connector.request_finished, "request_finished")
|
||||
sched.encoder_cache_manager = MagicMock()
|
||||
order.attach_mock(sched.encoder_cache_manager.free, "encoder_cache_free")
|
||||
sched.finished_req_ids = set()
|
||||
sched._new_prompt_len_snapshot = {}
|
||||
sched.finished_req_ids_dict = None
|
||||
sched._should_transfer_kv_for_request = lambda req_id: False
|
||||
sched._free_blocks = MagicMock()
|
||||
sched._free_input_coordinator_request = MagicMock()
|
||||
sched.chunk_transfer_adapter = None
|
||||
return sched, order
|
||||
|
||||
|
||||
class _FakeFinishedRequest:
|
||||
def __init__(self, request_id: str) -> None:
|
||||
self.request_id = request_id
|
||||
|
||||
def is_finished(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def test_free_request_fires_ec_hook_before_encoder_cache_free() -> None:
|
||||
ec_connector = MagicMock()
|
||||
ec_connector.request_finished.return_value = (False, EC_PARAMS)
|
||||
sched, order = _make_free_request_sched(ec_connector=ec_connector)
|
||||
request = _FakeFinishedRequest("req-ec")
|
||||
|
||||
kv_params, ec_params = sched._free_request(request)
|
||||
|
||||
assert ec_params == EC_PARAMS
|
||||
ec_connector.request_finished.assert_called_once_with(request)
|
||||
hook_calls = [name for name, *_ in order.mock_calls]
|
||||
assert hook_calls.index("request_finished") < hook_calls.index("encoder_cache_free")
|
||||
sched._free_blocks.assert_called_once_with(request)
|
||||
|
||||
|
||||
def test_free_request_honors_ec_delayed_free() -> None:
|
||||
ec_connector = MagicMock()
|
||||
ec_connector.request_finished.return_value = (True, EC_PARAMS)
|
||||
sched, _ = _make_free_request_sched(ec_connector=ec_connector)
|
||||
request = _FakeFinishedRequest("req-ec-delay")
|
||||
|
||||
kv_params, ec_params = sched._free_request(request)
|
||||
|
||||
assert ec_params == EC_PARAMS
|
||||
sched._free_blocks.assert_not_called()
|
||||
|
||||
|
||||
def test_free_request_without_ec_connector_returns_none() -> None:
|
||||
sched, _ = _make_free_request_sched(ec_connector=None)
|
||||
request = _FakeFinishedRequest("req-no-ec")
|
||||
|
||||
kv_params, ec_params = sched._free_request(request)
|
||||
|
||||
assert ec_params is None
|
||||
sched._free_blocks.assert_called_once_with(request)
|
||||
|
||||
|
||||
def _make_finishing_request() -> Request:
|
||||
request = Request(
|
||||
request_id="req-ec-finish",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=4),
|
||||
pooling_params=None,
|
||||
arrival_time=100.0,
|
||||
block_hasher=None,
|
||||
)
|
||||
request.status = RequestStatus.RUNNING
|
||||
return request
|
||||
|
||||
|
||||
def _make_finish_sched(request: Request) -> MagicMock:
|
||||
"""MagicMock scheduler driving the real ``update_from_output`` through a
|
||||
finishing request; ``_free_request`` reports EC transfer params."""
|
||||
sched = MagicMock()
|
||||
sched.requests = {request.request_id: request}
|
||||
sched.perf_metrics = None
|
||||
sched.defer_block_free = False
|
||||
sched.structured_output_manager.should_advance.return_value = False
|
||||
sched._process_kv_transfer_trigger.return_value = False
|
||||
sched._maybe_decode_pooling_output.return_value = None
|
||||
sched._handle_stopped_request.return_value = True
|
||||
sched._free_request.return_value = (None, EC_PARAMS)
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.finished_req_ids_dict = None
|
||||
sched._new_prompt_len_snapshot = {}
|
||||
return sched
|
||||
|
||||
|
||||
def test_ar_update_from_output_carries_ec_params_to_engine_output() -> None:
|
||||
request = _make_finishing_request()
|
||||
sched = _make_finish_sched(request)
|
||||
sched._update_request_with_output.return_value = ([42], True)
|
||||
|
||||
engine_core_outputs = OmniARScheduler.update_from_output(sched, *_finish_frame_parts(request))
|
||||
|
||||
(eco,) = engine_core_outputs[request.client_index].outputs
|
||||
assert eco.ec_transfer_params == EC_PARAMS
|
||||
sched._free_request.assert_called_once_with(request)
|
||||
|
||||
|
||||
def _finish_frame_parts(request: Request):
|
||||
scheduler_output = MagicMock(spec=SchedulerOutput)
|
||||
scheduler_output.num_scheduled_tokens = {request.request_id: 1}
|
||||
scheduler_output.total_num_scheduled_tokens = 1
|
||||
scheduler_output.scheduled_spec_decode_tokens = {}
|
||||
scheduler_output.num_invalid_spec_tokens = 0
|
||||
|
||||
model_runner_output = MagicMock(spec=ModelRunnerOutput)
|
||||
model_runner_output.sampled_token_ids = [[42]]
|
||||
model_runner_output.logprobs = None
|
||||
model_runner_output.prompt_logprobs_dict = {}
|
||||
model_runner_output.pooler_output = None
|
||||
model_runner_output.num_nans_in_logits = None
|
||||
model_runner_output.kv_connector_output = None
|
||||
model_runner_output.cudagraph_stats = None
|
||||
model_runner_output.req_id_to_index = {request.request_id: 0}
|
||||
model_runner_output.routed_experts = None
|
||||
return scheduler_output, model_runner_output
|
||||
|
||||
|
||||
def test_generation_update_from_output_carries_ec_params_to_engine_output() -> None:
|
||||
request = _make_finishing_request()
|
||||
# One-shot generation finish: the whole prompt is computed.
|
||||
request.num_computed_tokens = len(request.prompt_token_ids)
|
||||
sched = _make_finish_sched(request)
|
||||
|
||||
engine_core_outputs = OmniGenerationScheduler.update_from_output(sched, *_finish_frame_parts(request))
|
||||
|
||||
(eco,) = engine_core_outputs[request.client_index].outputs
|
||||
assert eco.ec_transfer_params == EC_PARAMS
|
||||
sched._free_request.assert_called_once_with(request)
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""v0.28 prefill-stats finalization in the Omni schedulers (#6606 review).
|
||||
|
||||
Upstream v0.28 finalizes prefill statistics with
|
||||
``PrefillStats.finalize(kv_cache_manager.estimate_cached_tokens(request))``
|
||||
before emitting them, which computes ``num_cache_creation_tokens`` (tokens
|
||||
computed and written to the prefix cache). Both Omni schedulers previously
|
||||
emitted ``take_prefill_stats()`` raw, so creation usage was always 0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request
|
||||
# before Request / RequestStatus are bound in this module.
|
||||
# isort: off
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
|
||||
# isort: on
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_request() -> Request:
|
||||
request = Request(
|
||||
request_id="req-prefill-stats",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=4),
|
||||
pooling_params=None,
|
||||
arrival_time=100.0,
|
||||
block_hasher=None,
|
||||
)
|
||||
request.status = RequestStatus.RUNNING
|
||||
# A scheduled prefill: 3 prompt tokens, 1 from the local prefix cache.
|
||||
request.prefill_stats.set(
|
||||
num_prompt_tokens=3,
|
||||
num_local_cached_tokens=1,
|
||||
num_external_cached_tokens=0,
|
||||
)
|
||||
return request
|
||||
|
||||
|
||||
def _make_sched(request: Request) -> MagicMock:
|
||||
sched = MagicMock()
|
||||
sched.requests = {request.request_id: request}
|
||||
sched.perf_metrics = None
|
||||
sched.defer_block_free = False
|
||||
sched.structured_output_manager.should_advance.return_value = False
|
||||
sched._process_kv_transfer_trigger.return_value = False
|
||||
sched._maybe_decode_pooling_output.return_value = None
|
||||
sched._handle_stopped_request.return_value = True
|
||||
# The whole 3-token prompt ends up cached: 3 - 1 pre-cached = 2 created.
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 3
|
||||
|
||||
def _free_and_invalidate(req):
|
||||
# Lifecycle guard: freeing releases the KV blocks, after which the
|
||||
# manager reports 0 cached tokens. If finalize ran after
|
||||
# _free_request (the reviewed bug), creation count would compute
|
||||
# against 0 and the assertions below would see 0, not 2.
|
||||
sched.kv_cache_manager.estimate_cached_tokens.return_value = 0
|
||||
return (None, None)
|
||||
|
||||
sched._free_request.side_effect = _free_and_invalidate
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.finished_req_ids_dict = None
|
||||
sched._new_prompt_len_snapshot = {}
|
||||
return sched
|
||||
|
||||
|
||||
def _frame(request: Request):
|
||||
scheduler_output = MagicMock(spec=SchedulerOutput)
|
||||
scheduler_output.num_scheduled_tokens = {request.request_id: 1}
|
||||
scheduler_output.total_num_scheduled_tokens = 1
|
||||
scheduler_output.scheduled_spec_decode_tokens = {}
|
||||
scheduler_output.num_invalid_spec_tokens = 0
|
||||
|
||||
model_runner_output = MagicMock(spec=ModelRunnerOutput)
|
||||
model_runner_output.sampled_token_ids = [[42]]
|
||||
model_runner_output.logprobs = None
|
||||
model_runner_output.prompt_logprobs_dict = {}
|
||||
model_runner_output.pooler_output = None
|
||||
model_runner_output.num_nans_in_logits = None
|
||||
model_runner_output.kv_connector_output = None
|
||||
model_runner_output.cudagraph_stats = None
|
||||
model_runner_output.req_id_to_index = {request.request_id: 0}
|
||||
model_runner_output.routed_experts = None
|
||||
return scheduler_output, model_runner_output
|
||||
|
||||
|
||||
def test_ar_scheduler_finalizes_prefill_stats_before_emitting() -> None:
|
||||
request = _make_request()
|
||||
sched = _make_sched(request)
|
||||
sched._update_request_with_output.return_value = ([42], True)
|
||||
|
||||
engine_core_outputs = OmniARScheduler.update_from_output(sched, *_frame(request))
|
||||
|
||||
(eco,) = engine_core_outputs[request.client_index].outputs
|
||||
assert eco.prefill_stats is not None
|
||||
assert eco.prefill_stats.num_cached_tokens == 1
|
||||
assert eco.prefill_stats.num_cache_creation_tokens == 2
|
||||
sched.kv_cache_manager.estimate_cached_tokens.assert_called_once_with(request)
|
||||
|
||||
|
||||
def test_generation_scheduler_finalizes_prefill_stats_before_emitting() -> None:
|
||||
request = _make_request()
|
||||
request.num_computed_tokens = len(request.prompt_token_ids)
|
||||
sched = _make_sched(request)
|
||||
|
||||
engine_core_outputs = OmniGenerationScheduler.update_from_output(sched, *_frame(request))
|
||||
|
||||
(eco,) = engine_core_outputs[request.client_index].outputs
|
||||
assert eco.prefill_stats is not None
|
||||
assert eco.prefill_stats.num_cached_tokens == 1
|
||||
assert eco.prefill_stats.num_cache_creation_tokens == 2
|
||||
sched.kv_cache_manager.estimate_cached_tokens.assert_called_once_with(request)
|
||||
Reference in new issue
Block a user