[Rebase] Align vLLM-Omni with vLLM v0.31.0 (#8459)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
1f47fa2f29
commit
0b0d2d7009
141 files changed
+2885
-721
No files matched your search
@@ -44,8 +44,8 @@ def _check_prefill():
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=128, dtype=torch.bfloat16)
|
||||
kvconfig = KVCacheConfig(num_blocks=1024, kv_cache_tensors=[], kv_cache_groups=[KVCacheGroupSpec(["attn"], spec)])
|
||||
state = OmniModelState.__new__(OmniModelState)
|
||||
state.model_config = SimpleNamespace(max_model_len=256)
|
||||
state.vllm_config = c
|
||||
state.max_model_len = 256
|
||||
state.supports_mm_inputs = False
|
||||
with set_current_vllm_config(c):
|
||||
group = AttentionGroup(FlashAttentionBackend, ["attn"], spec, 0)
|
||||
|
||||
@@ -54,7 +54,7 @@ def test_full_decode_graph_matches_fa3_aot_schedule(graph_mode, tmp_path):
|
||||
kvconfig = KVCacheConfig(num_blocks=1024, kv_cache_tensors=[], kv_cache_groups=[KVCacheGroupSpec(["attn"], spec)])
|
||||
state = OmniModelState.__new__(OmniModelState)
|
||||
state.vllm_config = c
|
||||
state.max_model_len = 2048
|
||||
state.model_config = c.model_config
|
||||
state.supports_mm_inputs = False
|
||||
with set_current_vllm_config(c):
|
||||
group = AttentionGroup(FlashAttentionBackend, ["attn"], spec, 0)
|
||||
|
||||
@@ -11,10 +11,13 @@ from unittest.mock import MagicMock
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.v1.outputs import RoutedExpertsTensors
|
||||
from vllm.distributed.aux_output_connector.connector import AuxRequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
|
||||
from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
|
||||
|
||||
import vllm_omni.worker_v2.omni_ar_model_runner as omni_ar_model_runner
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.model_executor.output_snapshot import PackedOutputSnapshot
|
||||
from vllm_omni.worker_v2.omni_ar_model_runner import OmniARModelRunner, OmniAsyncOutput
|
||||
from vllm_omni.worker_v2.output_snapshot import pack_output_snapshot
|
||||
@@ -41,7 +44,7 @@ class _FakeEvent:
|
||||
def _async_output(req_ids=("req-0",), **overrides) -> OmniAsyncOutput:
|
||||
rid2idx = {rid: i for i, rid in enumerate(req_ids)}
|
||||
mro = omni_ar_model_runner.OmniModelRunnerOutput(list(req_ids), rid2idx, None, prompt_logprobs_dict={})
|
||||
sampler_output = SamplerOutput(torch.tensor([[123]]), None, None, None)
|
||||
sampler_output = SamplerOutput(torch.tensor([[123]]), None, None, torch.tensor([1]), torch.tensor([0]))
|
||||
kwargs = dict(model_runner_output=mro, sampler_output=sampler_output)
|
||||
kwargs.update(num_sampled_tokens=torch.tensor([1] * len(req_ids)), copy_event=_FakeEvent())
|
||||
kwargs.update(main_stream=_FakeStream(), copy_stream=_FakeStream())
|
||||
@@ -64,23 +67,38 @@ def test_async_output_blocking_event_and_routing_masks(monkeypatch, compact_widt
|
||||
counts=torch.tensor([2, 0]),
|
||||
vocab_size=4,
|
||||
)
|
||||
sampler_output = SamplerOutput(torch.tensor([[2], [0]]), None, None, torch.tensor([1, 0]), None, masks)
|
||||
routed = RoutedExpertsTensors(torch.tensor([[[2, 3]], [[4, 5]]]), torch.tensor([7, 9]))
|
||||
sampler_output = SamplerOutput(
|
||||
torch.tensor([[2], [0]]), None, None, torch.tensor([1, 0]), torch.tensor([0, 0]), masks
|
||||
)
|
||||
copied: list[tuple[np.ndarray, np.ndarray]] = []
|
||||
|
||||
class _FakePendingAuxOutput:
|
||||
def enqueue_cpu_copy(self, *, num_sampled, num_rejected) -> None:
|
||||
copied.append((num_sampled, num_rejected))
|
||||
|
||||
def process_output(self) -> dict[str, AuxRequestOutput]:
|
||||
return {
|
||||
"decode": AuxRequestOutput(token_start=0, rows=np.array([[2, 3]], dtype=np.uint8)),
|
||||
"prefill": AuxRequestOutput(token_start=0, rows=np.array([[4, 5]], dtype=np.uint8)),
|
||||
}
|
||||
|
||||
pending = _FakePendingAuxOutput()
|
||||
output = _async_output(
|
||||
req_ids=["decode", "prefill"],
|
||||
sampler_output=sampler_output,
|
||||
num_sampled_tokens=torch.tensor([1, 0]),
|
||||
copy_event=None, # None → constructor builds the default blocking event
|
||||
routed_experts=routed,
|
||||
pending_aux_output=pending,
|
||||
).get_output()
|
||||
assert event_kwargs == [{"blocking": True}] # blocking event by default
|
||||
assert output.sampled_token_ids == [[2], []]
|
||||
assert output.sampling_masks.to_nested_list() == [[0, 2], []]
|
||||
np.testing.assert_array_equal(output.sampling_masks.offsets, [0, 2, 2])
|
||||
np.testing.assert_array_equal(output.routed_experts.routing_data, [[[2, 3]], [[4, 5]]])
|
||||
np.testing.assert_array_equal(output.routed_experts.slot_mapping, [7, 9])
|
||||
np.testing.assert_array_equal(output.aux_output_connector_output["decode"].rows, [[2, 3]])
|
||||
np.testing.assert_array_equal(output.aux_output_connector_output["prefill"].rows, [[4, 5]])
|
||||
assert output.aux_output_connector_output["decode"].token_start == 0
|
||||
np.testing.assert_array_equal(output.sampling_masks.token_ids, [0, 2])
|
||||
assert len(copied) == 1 and copied[0][0].tolist() == [1, 0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("needs_history", [False, True])
|
||||
@@ -88,19 +106,26 @@ def test_last_pp_rank_orchestration_and_kv_resolver(monkeypatch, needs_history)
|
||||
runner = OmniARModelRunner.__new__(OmniARModelRunner)
|
||||
input_batch = SimpleNamespace(req_ids=["req"], num_reqs=1, seq_lens=torch.tensor([3]))
|
||||
input_batch.idx_mapping, input_batch.query_start_loc = torch.tensor([0]), torch.tensor([0, 1])
|
||||
input_batch.idx_mapping_np = np.array([0])
|
||||
input_batch.num_computed_prefill_tokens_np = np.array([0])
|
||||
input_batch.num_scheduled_tokens = np.array([3])
|
||||
input_batch.prefill_len_np = np.array([3])
|
||||
input_batch.query_start_loc_np = np.array([0, 3])
|
||||
state = SimpleNamespace(input_batch=input_batch, hidden_states=torch.zeros(1, 2))
|
||||
state.finished_req_ids, state.ec_connector_output, state.routed_experts = {"finished"}, None, None
|
||||
state.finished_req_ids, state.ec_connector_output = {"finished"}, None
|
||||
runner.execute_model_state = state
|
||||
runner._kv_extracted_req_ids = runner._last_aux_output = runner._last_multimodal_outputs = None
|
||||
runner.is_last_pp_rank, runner.pp_handler, runner.check_ep_fault = True, None, False
|
||||
runner.aux_output_connector = None
|
||||
runner.model_config = SimpleNamespace(async_chunk=False)
|
||||
runner.vllm_config = SimpleNamespace(model_config=SimpleNamespace(engine_output_type="text"))
|
||||
runner.model_state = SimpleNamespace(postprocess_model_output=MagicMock(return_value=(torch.zeros(1, 2), None)))
|
||||
text_hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
runner.model_state = SimpleNamespace(postprocess_model_output=MagicMock(return_value=(text_hidden, None)))
|
||||
runner.model_state.intermediate_buffer = SimpleNamespace(buffers={0: {"global_request_id": "global-req"}})
|
||||
runner.req_states = SimpleNamespace(req_id_to_index={"req": 0})
|
||||
runner.req_states.all_token_ids = SimpleNamespace(gpu=torch.tensor([[1]]))
|
||||
runner.req_states.num_computed_tokens = SimpleNamespace(gpu=torch.tensor([0]))
|
||||
runner.req_states.prompt_len = SimpleNamespace(np=np.array([1]), gpu=torch.tensor([1]))
|
||||
runner.req_states.prompt_len = SimpleNamespace(np=np.array([3]), gpu=torch.tensor([3]))
|
||||
runner.main_stream = runner.output_copy_stream = MagicMock()
|
||||
runner.eplb = runner._finalize_native_data_plane_output = runner._reserve_native_data_plane_outputs = MagicMock()
|
||||
sampler_out = (SimpleNamespace(sampled_token_ids=torch.tensor([[2]])), MagicMock(), MagicMock())
|
||||
@@ -114,13 +139,17 @@ def test_last_pp_rank_orchestration_and_kv_resolver(monkeypatch, needs_history)
|
||||
yield
|
||||
sampling_active = False
|
||||
|
||||
runner.model = SimpleNamespace(compute_logits=None, logitsprocs_need_output_token_ids=needs_history)
|
||||
runner.model = SimpleNamespace(
|
||||
compute_logits=lambda hidden: hidden, logitsprocs_need_output_token_ids=needs_history
|
||||
)
|
||||
runner.model.mrv2_sampling_context = sampling_context
|
||||
runner.sampler = None
|
||||
runner.sample = MagicMock(return_value=sampler_out)
|
||||
runner.sample.side_effect = lambda *_: sampler_out if sampling_active is needs_history else pytest.fail()
|
||||
logprobs_mock = MagicMock(side_effect=lambda *_: pytest.fail("inside ctx") if sampling_active else {})
|
||||
runner.prompt_logprobs_worker = SimpleNamespace(compute_prompt_logprobs=logprobs_mock)
|
||||
runner.prompt_logprobs_worker = PromptLogprobsWorker(1, torch.device("cpu"), logprobs_mode="raw_logits")
|
||||
runner.prompt_logprobs_worker.add_request("req", 0, SamplingParams(prompt_logprob_token_ids=[1, 0]))
|
||||
runner.prompt_logprobs_worker.compute_prompt_logprobs = logprobs_mock
|
||||
runner.postprocess_sampled, connector_output = MagicMock(), object()
|
||||
|
||||
def post_forward(finished_req_ids):
|
||||
@@ -135,10 +164,67 @@ def test_last_pp_rank_orchestration_and_kv_resolver(monkeypatch, needs_history)
|
||||
assert runner.sample_tokens(None) is mock_out
|
||||
built = omni_ar_model_runner.OmniAsyncOutput.call_args.kwargs["model_runner_output"]
|
||||
assert built.kv_connector_output is connector_output
|
||||
torch.testing.assert_close(built.prompt_token_id_logprobs_dict["req"], torch.tensor([[2.0, 1.0], [4.0, 3.0]]))
|
||||
assert runner._resolve_global_request_id("req") == "global-req" # from the intermediate buffer
|
||||
assert runner._resolve_global_request_id("unknown") == "unknown" # fallback to the local id
|
||||
|
||||
|
||||
def test_async_output_preserves_fixed_token_scores_on_host(monkeypatch) -> None:
|
||||
monkeypatch.setattr(torch.cuda, "set_stream", lambda _stream: None)
|
||||
scores = torch.tensor([[-0.5, -1.5], [-0.2, -2.2]])
|
||||
mro = omni_ar_model_runner.OmniModelRunnerOutput(
|
||||
req_ids=["req-0"],
|
||||
req_id_to_index={"req-0": 0},
|
||||
sampled_token_ids=None,
|
||||
prompt_logprobs_dict={},
|
||||
prompt_token_id_logprobs_dict={"req-0": scores},
|
||||
)
|
||||
output = _async_output(model_runner_output=mro).get_output()
|
||||
assert output.prompt_token_id_logprobs_dict["req-0"].device.type == "cpu"
|
||||
torch.testing.assert_close(output.prompt_token_id_logprobs_dict["req-0"], scores)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_chunked_fixed_token_scores_are_copied_from_cuda() -> None:
|
||||
device = torch.device("cuda")
|
||||
hidden = torch.tensor([[1.0, 2.0, 3.0], [3.0, 1.0, 2.0], [2.0, 3.0, 1.0], [4.0, 5.0, 6.0]], device=device)
|
||||
worker = PromptLogprobsWorker(1, device)
|
||||
worker.add_request("req", 0, SamplingParams(prompt_logprob_token_ids=[2, 0], prompt_logprob_start=1))
|
||||
batch = SimpleNamespace(
|
||||
req_ids=["req"],
|
||||
idx_mapping_np=np.array([0]),
|
||||
num_computed_prefill_tokens_np=np.array([0]),
|
||||
num_scheduled_tokens=np.array([2]),
|
||||
prefill_len_np=np.array([4]),
|
||||
query_start_loc_np=np.array([0, 2]),
|
||||
)
|
||||
assert worker.compute_prompt_token_id_logprobs(lambda hidden: hidden, hidden[:2], batch, np.array([4])) == {}
|
||||
batch.num_computed_prefill_tokens_np = np.array([2])
|
||||
scores = worker.compute_prompt_token_id_logprobs(lambda hidden: hidden, hidden[2:], batch, np.array([4]))
|
||||
mro = omni_ar_model_runner.OmniModelRunnerOutput(
|
||||
req_ids=["req"],
|
||||
req_id_to_index={"req": 0},
|
||||
sampled_token_ids=None,
|
||||
prompt_logprobs_dict={},
|
||||
prompt_token_id_logprobs_dict=scores,
|
||||
)
|
||||
num_sampled = torch.tensor([1], device=device, dtype=torch.int32)
|
||||
sampler_output = SamplerOutput(
|
||||
torch.tensor([[1]], device=device), None, None, num_sampled, torch.zeros_like(num_sampled)
|
||||
)
|
||||
output = OmniAsyncOutput(
|
||||
model_runner_output=mro,
|
||||
sampler_output=sampler_output,
|
||||
num_sampled_tokens=num_sampled,
|
||||
main_stream=torch.cuda.current_stream(),
|
||||
copy_stream=torch.cuda.Stream(),
|
||||
).get_output()
|
||||
|
||||
expected = hidden[1:3].log_softmax(dim=-1)[:, [2, 0]].cpu()
|
||||
assert output.prompt_token_id_logprobs_dict["req"].device.type == "cpu"
|
||||
torch.testing.assert_close(output.prompt_token_id_logprobs_dict["req"], expected)
|
||||
|
||||
|
||||
def test_async_mm_snapshot_owns_output_until_copy_finishes() -> None:
|
||||
runner = OmniARModelRunner.__new__(OmniARModelRunner)
|
||||
runner.model_config = SimpleNamespace(async_chunk=True)
|
||||
@@ -284,7 +370,13 @@ def test_async_output_slices_request_payloads_with_graph_padding(
|
||||
batch.num_tokens_after_padding = padded_total
|
||||
output = _async_output(
|
||||
req_ids=[f"req-{i}" for i in range(len(lengths))],
|
||||
sampler_output=SamplerOutput(torch.ones(len(lengths), 1, dtype=torch.long), None, None, None),
|
||||
sampler_output=SamplerOutput(
|
||||
torch.ones(len(lengths), 1, dtype=torch.long),
|
||||
None,
|
||||
None,
|
||||
torch.ones(len(lengths), dtype=torch.long),
|
||||
torch.zeros(len(lengths), dtype=torch.long),
|
||||
),
|
||||
text_hidden=hidden,
|
||||
multimodal_outputs={"codes": {"audio": codes, "ref": refs}},
|
||||
input_batch=batch,
|
||||
@@ -585,7 +677,9 @@ def test_model_owned_audio_finalizer_runs_after_copy_without_hidden(monkeypatch,
|
||||
|
||||
result = _async_output(
|
||||
req_ids=["audio", "partial"],
|
||||
sampler_output=SamplerOutput(torch.tensor([[99], [100]]), None, None, None),
|
||||
sampler_output=SamplerOutput(
|
||||
torch.tensor([[99], [100]]), None, None, torch.tensor([1, 1]), torch.tensor([0, 0])
|
||||
),
|
||||
num_sampled_tokens=torch.tensor([1, 0]),
|
||||
multimodal_outputs={"snapshot": snapshot},
|
||||
input_batch=batch,
|
||||
@@ -710,7 +804,7 @@ def test_stream_audio_snapshot_skips_discarded_code_partition(monkeypatch, mocke
|
||||
extra = ({"extra": torch.tensor([7])}, _FakeEvent()) if consumer == "extra" else None
|
||||
result = _async_output(
|
||||
req_ids=["frame", "eos", "prefill"],
|
||||
sampler_output=SamplerOutput(torch.tensor([[1], [2], [3]]), None, None, None),
|
||||
sampler_output=SamplerOutput(torch.tensor([[1], [2], [3]]), None, None, None, None),
|
||||
num_sampled_tokens=torch.tensor([1, 1, 1]),
|
||||
multimodal_outputs={"codes": {"audio": torch.ones(4, 2)}},
|
||||
input_batch=batch,
|
||||
@@ -753,7 +847,9 @@ def test_request_owned_snapshot_skips_generic_partition(monkeypatch, streaming):
|
||||
|
||||
result = _async_output(
|
||||
req_ids=["audio", "partial"],
|
||||
sampler_output=SamplerOutput(torch.tensor([[99], [100]]), None, None, None),
|
||||
sampler_output=SamplerOutput(
|
||||
torch.tensor([[99], [100]]), None, None, torch.tensor([1, 1]), torch.tensor([0, 0])
|
||||
),
|
||||
num_sampled_tokens=torch.tensor([1, 0]),
|
||||
multimodal_outputs={"snapshot": snapshot},
|
||||
input_batch=batch,
|
||||
|
||||
@@ -4,12 +4,14 @@
|
||||
"""OmniGenerationModelRunner contracts: empty-step lifecycle, output partition,
|
||||
CPU-sync vs CUDA-async dispatch, and async-chunk slot recycling."""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.distributed.aux_output_connector.connector import AuxRequestOutput
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.outputs import OmniModelRunnerOutput
|
||||
@@ -18,6 +20,17 @@ from vllm_omni.worker_v2.omni_generation_model_runner import OmniGenerationModel
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("randomize_inputs", [False, True])
|
||||
def test_profile_run_accepts_release_flag_without_running_codec(monkeypatch, randomize_inputs):
|
||||
runner = object.__new__(OmniGenerationModelRunner)
|
||||
runner.model = MagicMock(side_effect=AssertionError("profiling ran codec model"))
|
||||
synchronize = MagicMock()
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", synchronize)
|
||||
runner.profile_run(randomize_inputs=randomize_inputs)
|
||||
synchronize.assert_called_once_with()
|
||||
runner.model.assert_not_called()
|
||||
|
||||
|
||||
class _FakeStagedField:
|
||||
def __init__(self, data):
|
||||
self.np = data
|
||||
@@ -45,6 +58,7 @@ def _make_runner(model_output, num_reqs=1, prompt_len=10):
|
||||
runner.execute_model_state = SimpleNamespace(finished_req_ids={"finished"}, ec_connector_output=None)
|
||||
runner.kv_connector = SimpleNamespace(post_forward=MagicMock(return_value=None))
|
||||
runner.check_ep_fault = False
|
||||
runner.aux_output_connector = None
|
||||
runner.req_states = MagicMock(
|
||||
prompt_len=SimpleNamespace(np=np.full(num_reqs, prompt_len, dtype=np.int32)),
|
||||
num_computed_tokens=_FakeStagedField(np.zeros(num_reqs, dtype=np.int32)),
|
||||
@@ -52,7 +66,8 @@ def _make_runner(model_output, num_reqs=1, prompt_len=10):
|
||||
return runner
|
||||
|
||||
|
||||
def test_control_only_step_keeps_lifecycle_and_skips_input_construction():
|
||||
@pytest.mark.parametrize("aux_enabled", [False, True])
|
||||
def test_control_only_step_keeps_lifecycle_and_skips_input_construction(aux_enabled):
|
||||
runner = object.__new__(OmniGenerationModelRunner)
|
||||
order = []
|
||||
for name in (
|
||||
@@ -68,8 +83,13 @@ def test_control_only_step_keeps_lifecycle_and_skips_input_construction():
|
||||
runner.kv_connector = SimpleNamespace(no_forward=lambda _s: output)
|
||||
runner._merge_ec_connector_no_forward = lambda _s, value: value
|
||||
runner._attach_native_data_plane_signals = lambda value: value
|
||||
metadata = object()
|
||||
runner.aux_output_connector = SimpleNamespace(begin_step=MagicMock()) if aux_enabled else None
|
||||
scheduler_output = SimpleNamespace(
|
||||
total_num_scheduled_tokens=0, scheduled_new_reqs=[], scheduled_cached_reqs=SimpleNamespace(req_ids=[])
|
||||
total_num_scheduled_tokens=0,
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(req_ids=[]),
|
||||
aux_output_connector_metadata=metadata,
|
||||
)
|
||||
assert runner.execute_model(scheduler_output) is output
|
||||
assert order == [
|
||||
@@ -78,6 +98,9 @@ def test_control_only_step_keeps_lifecycle_and_skips_input_construction():
|
||||
"free_states",
|
||||
"_apply_block_table_staged_writes_if_available",
|
||||
]
|
||||
if aux_enabled:
|
||||
assert runner.aux_output_connector is not None
|
||||
runner.aux_output_connector.begin_step.assert_called_once_with(metadata)
|
||||
|
||||
|
||||
def test_released_chunk_reuses_scheduler_output_and_slot_recycle_clears_state():
|
||||
@@ -184,6 +207,72 @@ def test_sample_tokens_uses_async_output_for_cuda_and_snapshots_req_ids(monkeypa
|
||||
assert output_req_ids == ["req-0"]
|
||||
|
||||
|
||||
def test_generation_aux_output_copies_and_materializes_without_sampling(monkeypatch):
|
||||
from vllm_omni.worker_v2 import omni_generation_model_runner as module
|
||||
|
||||
events = []
|
||||
|
||||
class PendingOutput:
|
||||
def enqueue_cpu_copy(self, *, num_sampled, num_rejected):
|
||||
np.testing.assert_array_equal(num_sampled, [0])
|
||||
np.testing.assert_array_equal(num_rejected, [0])
|
||||
events.append("copy")
|
||||
|
||||
def process_output(self):
|
||||
assert events == ["copy", "record", "synchronize"]
|
||||
return {"req-0": AuxRequestOutput(0, np.array([[2, 3]], dtype=np.uint8))}
|
||||
|
||||
class CopyEvent:
|
||||
def record(self, stream):
|
||||
events.append("record")
|
||||
|
||||
def synchronize(self):
|
||||
events.append("synchronize")
|
||||
|
||||
monkeypatch.setattr(torch.cuda, "Event", lambda **kw: CopyEvent())
|
||||
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
|
||||
monkeypatch.setattr(module, "_async_copy_mm", lambda *a, **kw: {})
|
||||
output = OmniModelRunnerOutput(["req-0"], {"req-0": 0}, [[]])
|
||||
result = module.OmniGenerationAsyncOutput(
|
||||
model_runner_output=output,
|
||||
multimodal_outputs={},
|
||||
num_reqs=1,
|
||||
main_stream=object(),
|
||||
copy_stream=SimpleNamespace(wait_stream=lambda stream: None),
|
||||
pending_aux_output=PendingOutput(),
|
||||
).get_output()
|
||||
|
||||
assert result.sampled_token_ids == [[]]
|
||||
np.testing.assert_array_equal(result.aux_output_connector_output["req-0"].rows, [[2, 3]])
|
||||
|
||||
|
||||
def test_pending_aux_output_forces_async_delivery_without_cuda_payload(monkeypatch):
|
||||
from vllm_omni.worker_v2 import omni_generation_model_runner as module
|
||||
|
||||
runner = _make_runner(OmniOutput(text_hidden_states=torch.empty(0), multimodal_outputs={}))
|
||||
runner.device = SimpleNamespace(type="cuda")
|
||||
runner.main_stream = object()
|
||||
runner.output_copy_stream = object()
|
||||
pending = object()
|
||||
runner.aux_output_connector = SimpleNamespace(prepare_output=MagicMock(return_value=pending))
|
||||
runner._reserve_native_data_plane_outputs = MagicMock()
|
||||
runner._release_generation_slots = MagicMock()
|
||||
runner._finalize_native_data_plane_output = MagicMock()
|
||||
captured = {}
|
||||
|
||||
class AsyncOutput:
|
||||
def __init__(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
|
||||
monkeypatch.setattr(module, "OmniGenerationAsyncOutput", AsyncOutput)
|
||||
input_batch = runner._gen_input_batch
|
||||
result = runner.sample_tokens()
|
||||
|
||||
assert isinstance(result, AsyncOutput)
|
||||
runner.aux_output_connector.prepare_output.assert_called_once_with(input_batch)
|
||||
assert captured["pending_aux_output"] is pending
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exact", [True, False])
|
||||
def test_exact_input_shape_models_reject_padded_batches(exact):
|
||||
# Qwen3-Omni Code2Wav splits input_ids by seq_token_counts (#6712).
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm import SamplingParams
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.config.model import OmniModelConfig
|
||||
@@ -95,7 +96,8 @@ def test_full_payload_receive_is_polled_without_scheduled_tokens(mocker):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_form", ["tuple", "omni"])
|
||||
def test_capture_model_unwraps_exclude_full_and_capture_mtp(output_form):
|
||||
@pytest.mark.parametrize("profile_only", [False, True])
|
||||
def test_capture_model_unwraps_exclude_full_and_capture_mtp(output_form, profile_only):
|
||||
runner = object.__new__(OmniGPUModelRunner)
|
||||
hidden = torch.ones(1, 2)
|
||||
|
||||
@@ -117,16 +119,21 @@ def test_capture_model_unwraps_exclude_full_and_capture_mtp(output_form):
|
||||
runner.model_state = SimpleNamespace(capture_mtp_graphs=MagicMock())
|
||||
runner._dispatch_mtp_batch_descriptor = MagicMock(return_value="desc")
|
||||
|
||||
def assert_unwrapped(_self):
|
||||
def assert_unwrapped(_self, *, profile_only: bool = False):
|
||||
assert torch.equal(runner.model.forward(), hidden) # forward unwrapped during capture
|
||||
assert profile_only is expected_profile_only
|
||||
return 3
|
||||
|
||||
expected_profile_only = profile_only
|
||||
with patch.object(GPUModelRunner, "capture_model", assert_unwrapped):
|
||||
assert runner.capture_model() == 3
|
||||
assert runner.capture_model(profile_only=profile_only) == 3
|
||||
|
||||
assert runner.model.forward is original_forward # restored after capture
|
||||
assert runner.cudagraph_manager._capture_descs == {CUDAGraphMode.PIECEWISE: [piecewise]}
|
||||
runner.model_state.capture_mtp_graphs.assert_called_once_with(runner._dispatch_mtp_batch_descriptor)
|
||||
if profile_only:
|
||||
runner.model_state.capture_mtp_graphs.assert_not_called()
|
||||
else:
|
||||
runner.model_state.capture_mtp_graphs.assert_called_once_with(runner._dispatch_mtp_batch_descriptor)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -163,6 +170,28 @@ def test_init_model_state_factory_dispatches_omni_only(monkeypatch, flag, expect
|
||||
assert state is upstream.return_value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_stage", [False, True])
|
||||
@pytest.mark.parametrize("specialized", [False, True])
|
||||
def test_auxiliary_default_state_has_omni_lifecycle_but_preserves_upstream_states(monkeypatch, omni_stage, specialized):
|
||||
class SpecializedState(DefaultModelState):
|
||||
pass
|
||||
|
||||
state_cls = SpecializedState if specialized else DefaultModelState
|
||||
upstream_state = object.__new__(state_cls)
|
||||
monkeypatch.setattr("vllm_omni.worker_v2.model_states._upstream_init_model_state", lambda *_args: upstream_state)
|
||||
monkeypatch.setattr(OmniModelState, "__init__", lambda *_args: None)
|
||||
model_config = object.__new__(OmniModelConfig) if omni_stage else SimpleNamespace()
|
||||
state = init_omni_model_state(
|
||||
SimpleNamespace(model_config=model_config), SimpleNamespace(), None, torch.device("cpu")
|
||||
)
|
||||
if omni_stage and not specialized:
|
||||
assert isinstance(state, OmniModelState)
|
||||
assert callable(state.run_preprocess)
|
||||
assert callable(state.postprocess_model_output)
|
||||
else:
|
||||
assert state is upstream_state
|
||||
|
||||
|
||||
def test_finish_requests_notifies_model_and_cleans_only_known_slots(monkeypatch):
|
||||
runner = _make_runner()
|
||||
calls = []
|
||||
@@ -209,7 +238,7 @@ def test_full_graph_aux_contract_keeps_full_and_round_trips_leaves():
|
||||
runner.model_state = SimpleNamespace()
|
||||
captured = {}
|
||||
|
||||
def capture(_self):
|
||||
def capture(_self, *, profile_only=False):
|
||||
# The graph manager stores (hidden, leaves) in its aux buffers.
|
||||
assert runner.use_aux_hidden_state_outputs is True
|
||||
hidden, leaves = runner.model.forward(4)
|
||||
@@ -262,7 +291,9 @@ def test_full_graph_output_without_contract_is_hidden_only():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_kind", ["gpu", "ar", "generation"])
|
||||
def test_dummy_forward_uses_upstream_execution_state(runner_kind, monkeypatch):
|
||||
@pytest.mark.parametrize("randomize_inputs", [False, True])
|
||||
@pytest.mark.parametrize("is_profile", [False, True])
|
||||
def test_dummy_forward_uses_upstream_execution_state(runner_kind, randomize_inputs, is_profile, monkeypatch):
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_runner import ExecuteModelState
|
||||
|
||||
@@ -288,6 +319,7 @@ def test_dummy_forward_uses_upstream_execution_state(runner_kind, monkeypatch):
|
||||
runner.model_state.prepare_inputs.return_value = {}
|
||||
runner._omni_data_plane = object()
|
||||
runner.supports_mm_inputs = False
|
||||
runner.vocab_size = 32
|
||||
runner.lora_config = None
|
||||
runner.is_encoder_decoder = False
|
||||
runner.eplb = MagicMock()
|
||||
@@ -308,7 +340,15 @@ def test_dummy_forward_uses_upstream_execution_state(runner_kind, monkeypatch):
|
||||
cg_mode=CUDAGraphMode.NONE, num_reqs=1, num_tokens=1, num_active_loras=0, max_query_len=1
|
||||
)
|
||||
runner._dispatch_batch_descriptor = MagicMock(return_value=(batch_desc, None))
|
||||
monkeypatch.setattr(InputBatch, "make_dummy", lambda *args, **kwargs: input_batch)
|
||||
make_dummy = MagicMock(return_value=input_batch)
|
||||
monkeypatch.setattr(InputBatch, "make_dummy", make_dummy)
|
||||
randomized = []
|
||||
|
||||
def randomize(tensor, low, high):
|
||||
randomized.append((low, high))
|
||||
return tensor.fill_(7)
|
||||
|
||||
monkeypatch.setattr(torch.Tensor, "random_", randomize)
|
||||
monkeypatch.setattr("vllm_omni.worker_v2.omni_model_runner.build_slot_mappings_by_layer", lambda *args: {})
|
||||
for module in ("omni_model_runner", "omni_generation_model_runner"):
|
||||
monkeypatch.setattr(f"vllm_omni.worker_v2.{module}.set_forward_context", lambda *args, **kwargs: nullcontext())
|
||||
@@ -318,7 +358,20 @@ def test_dummy_forward_uses_upstream_execution_state(runner_kind, monkeypatch):
|
||||
|
||||
# Upstream _dummy_run always supplies valid_dummy_state_slots, and the
|
||||
# result must use the real upstream constructor rather than a mocked state.
|
||||
assert runner.execute_model(scheduled, dummy_run=True, valid_dummy_state_slots=True) is None
|
||||
assert (
|
||||
runner.execute_model(
|
||||
scheduled,
|
||||
dummy_run=True,
|
||||
valid_dummy_state_slots=True,
|
||||
randomize_inputs=randomize_inputs,
|
||||
is_profile=is_profile,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert make_dummy.call_args.kwargs["is_padding"] is not is_profile
|
||||
should_randomize = randomize_inputs and runner_kind != "generation"
|
||||
assert randomized == ([(0, 32)] if should_randomize else [])
|
||||
assert input_batch.input_ids.tolist() == ([7] if should_randomize else [1])
|
||||
assert isinstance(runner.execute_model_state, ExecuteModelState)
|
||||
assert runner.execute_model_state.input_batch is input_batch
|
||||
assert runner.execute_model_state.cudagraph_stats is None
|
||||
|
||||
@@ -22,6 +22,16 @@ from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState, _m
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_prepare_attn_forwards_release_model_specific_metadata():
|
||||
state = object.__new__(OmniModelState)
|
||||
metadata = object()
|
||||
output = object()
|
||||
batch = SimpleNamespace()
|
||||
with patch.object(DefaultModelState, "prepare_attn", return_value=output) as prepare:
|
||||
assert state.prepare_attn(batch, None, (), None, [], None, model_specific_attn_metadata=metadata) is output
|
||||
assert prepare.call_args.kwargs["model_specific_attn_metadata"] is metadata
|
||||
|
||||
|
||||
class _DummyInputBatch:
|
||||
is_prefilling_np: np.ndarray
|
||||
req_ids: list[str]
|
||||
|
||||
Reference in new issue
Block a user