[Bugfix] Restore BAGEL CFG KV transfers (#5620)
Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
This commit is contained in:
1 parent
185e3d6df6
commit
6f98c807d2
8 files changed
+160
-8
No files matched your search
@@ -0,0 +1,57 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from vllm import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.engine.async_engine_utils import apply_omni_final_stage_metadata
|
||||
from vllm_omni.engine.serialization import deserialize_additional_information
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_engine_request() -> EngineCoreRequest:
|
||||
return EngineCoreRequest(
|
||||
request_id="req",
|
||||
prompt_token_ids=[1],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
)
|
||||
|
||||
|
||||
def _request_omits_kv_transfer(*, force_kv_transfer: bool) -> tuple[bool, dict]:
|
||||
tagged = apply_omni_final_stage_metadata(
|
||||
_make_engine_request(),
|
||||
final_stage_id=0,
|
||||
force_kv_transfer=force_kv_transfer,
|
||||
)
|
||||
scheduler = OmniARScheduler.__new__(OmniARScheduler)
|
||||
scheduler._omits_kv_transfer_cache = {}
|
||||
request = SimpleNamespace(
|
||||
request_id="req",
|
||||
additional_information=tagged.additional_information,
|
||||
)
|
||||
result = scheduler._request_omits_kv_transfer_to_next_stage(request)
|
||||
metadata = deserialize_additional_information(tagged.additional_information)
|
||||
return result, metadata
|
||||
|
||||
|
||||
def test_stage_zero_request_omits_kv_transfer():
|
||||
omits_transfer, metadata = _request_omits_kv_transfer(force_kv_transfer=False)
|
||||
|
||||
assert omits_transfer
|
||||
assert "omni_force_kv_transfer" not in metadata
|
||||
|
||||
|
||||
def test_cfg_companion_forces_kv_transfer_without_downstream_payload():
|
||||
omits_transfer, metadata = _request_omits_kv_transfer(force_kv_transfer=True)
|
||||
|
||||
assert not omits_transfer
|
||||
assert metadata["omni_final_stage_id"] == 0
|
||||
assert metadata["omni_force_kv_transfer"] is True
|
||||
Reference in new issue
Block a user