Merge branch 'main' into qwen-image-2.1
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
51 files changed
+2189
-70
No files matched your search
@@ -67,11 +67,31 @@ def test_ready_diffusion_cpu_suite_is_sharded() -> None:
|
||||
assert "--shard-id=$$BUILDKITE_PARALLEL_JOB" in pytest_command
|
||||
|
||||
|
||||
def test_cosyvoice_gpu_abort_gets_one_fresh_job_retry() -> None:
|
||||
step = _find_step("CosyVoice3-TTS E2E Test", AMD_READY_PIPELINE)
|
||||
def test_cosyvoice_ready_smoke_uses_sdpa() -> None:
|
||||
step = _find_step("CosyVoice3-TTS E2E Smoke (SDPA)", AMD_READY_PIPELINE)
|
||||
|
||||
assert step["grade"] == "Blocking"
|
||||
assert step["retry"] == {"automatic": [{"exit_status": 134, "limit": 1}]}
|
||||
assert "export DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA" in step["commands"]
|
||||
|
||||
pytest_command = next(command for command in step["commands"] if "pytest" in command)
|
||||
assert "tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py::test_voice_clone_zh_002" in pytest_command
|
||||
|
||||
|
||||
def test_cosyvoice_full_default_backend_suite_runs_nightly() -> None:
|
||||
step = _find_step("CosyVoice3-TTS E2E Test", AMD_NIGHTLY_PIPELINE)
|
||||
|
||||
assert step["grade"] == "NonBlocking"
|
||||
assert step["timeout_in_minutes"] == 90
|
||||
assert step["retry"] == {"automatic": [{"exit_status": 134, "limit": 1}]}
|
||||
assert all("DIFFUSION_ATTENTION_BACKEND" not in command for command in step["commands"])
|
||||
|
||||
pytest_command = next(command for command in step["commands"] if "pytest" in command)
|
||||
assert "tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py" in pytest_command
|
||||
assert "::" not in pytest_command
|
||||
|
||||
|
||||
def test_amd_template_preserves_step_retry_policy() -> None:
|
||||
template = AMD_TEMPLATE.read_text(encoding="utf-8")
|
||||
# Both grouped and top-level AMD steps must preserve an explicit retry
|
||||
# policy when the source suite is rendered into the uploaded pipeline.
|
||||
|
||||
@@ -69,6 +69,39 @@ def _make_update(prompt_token_ids: list[int] | None = None) -> StreamingUpdate:
|
||||
)
|
||||
|
||||
|
||||
def _make_minicpm_window_update(
|
||||
*,
|
||||
seq: int,
|
||||
mode: str,
|
||||
high: int = 15,
|
||||
low: int = 10,
|
||||
context_max_units: int = 1,
|
||||
) -> StreamingUpdate:
|
||||
update = _make_update([0] * 8)
|
||||
update.model_intermediate_buffer = {
|
||||
"duplex": {
|
||||
"data_plane": True,
|
||||
"seq": seq,
|
||||
"runtime_config": {
|
||||
"duplex_scheduler_token_id": 0,
|
||||
"duplex_first_append_context_tokens": 3,
|
||||
"duplex_window_prefix_tokens": 2,
|
||||
"duplex_window_suffix_token_ids": [3],
|
||||
"duplex_window_previous_marker_token_ids": [70, 71],
|
||||
"duplex_window_special_token_ids": [99],
|
||||
"duplex_window_config": {
|
||||
"sliding_window_mode": mode,
|
||||
"basic_window_high_tokens": high,
|
||||
"basic_window_low_tokens": low,
|
||||
"context_previous_max_tokens": 4,
|
||||
"context_max_units": context_max_units,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return update
|
||||
|
||||
|
||||
def _make_talker_adapter(
|
||||
*,
|
||||
max_model_len: int = 100,
|
||||
@@ -510,6 +543,225 @@ def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder()
|
||||
assert sched._new_prompt_len_snapshot[session.request_id] == 2
|
||||
|
||||
|
||||
def test_stage0_basic_window_rebuilds_below_low_watermark() -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids.clear()
|
||||
session._all_token_ids.extend(session.prompt_token_ids)
|
||||
session.num_prompt_tokens = 9
|
||||
session.append_output_token_ids([40])
|
||||
session.num_computed_tokens = 10
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert session.prompt_token_ids == [0] * 9
|
||||
assert session.num_computed_tokens == 0
|
||||
assert update.model_intermediate_buffer["meta"]["replace_streaming_prompt"] is True
|
||||
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan == {
|
||||
"completed_token_ids": [40],
|
||||
"replace": True,
|
||||
"mode": "basic",
|
||||
"drop_units": 1,
|
||||
"dropped_tokens": 9,
|
||||
"previous_token_ids": [],
|
||||
"previous_marker_token_ids": [70, 71],
|
||||
"replacement_prompt_len": 9,
|
||||
}
|
||||
sched._free_request_blocks.assert_called_once_with(session)
|
||||
|
||||
|
||||
def test_stage0_context_window_compacts_dropped_speech() -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids.clear()
|
||||
session._all_token_ids.extend(session.prompt_token_ids)
|
||||
session.num_prompt_tokens = 9
|
||||
session.append_output_token_ids([40])
|
||||
session.num_computed_tokens = 10
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
|
||||
first = _make_minicpm_window_update(seq=2, mode="context")
|
||||
sched._update_request_as_session(session, first)
|
||||
assert session.num_prompt_tokens == 18
|
||||
assert first.model_intermediate_buffer["duplex"]["stage0_window"] == {"completed_token_ids": [40]}
|
||||
|
||||
session.append_output_token_ids([50])
|
||||
session.num_computed_tokens = 19
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
second = _make_minicpm_window_update(seq=3, mode="context")
|
||||
sched._update_request_as_session(session, second)
|
||||
|
||||
assert session.prompt_token_ids == [0] * 21
|
||||
plan = second.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan["completed_token_ids"] == [50]
|
||||
assert plan["drop_units"] == 1
|
||||
assert plan["previous_token_ids"] == [40]
|
||||
# The worker embeds this exact marker, not a re-tokenized one, so the
|
||||
# previous region is len(marker) + len(previous).
|
||||
assert plan["previous_marker_token_ids"] == [70, 71]
|
||||
assert plan["replacement_prompt_len"] == 21
|
||||
assert getattr(session, "_minicpmo45_window_previous_len") == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cleared_outputs", [False, True])
|
||||
@pytest.mark.parametrize("in_flight", [0, 1])
|
||||
def test_stage0_window_uses_confirmed_span_and_terminator(cleared_outputs, in_flight) -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids[:] = session.prompt_token_ids
|
||||
session.num_prompt_tokens = 9
|
||||
session.append_output_token_ids([40, 99])
|
||||
session.num_computed_tokens = 10 + in_flight
|
||||
session.num_output_placeholders = in_flight
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
if cleared_outputs:
|
||||
# A stop without a queued append clears this before the next update.
|
||||
session._output_token_ids.clear()
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan["completed_token_ids"] == [40]
|
||||
assert plan["completed_terminator_token_id"] == 99
|
||||
assert plan["replacement_prompt_len"] == 9
|
||||
assert plan["dropped_tokens"] == 9
|
||||
|
||||
|
||||
def test_stage0_window_rebuild_that_overflows_max_model_len_finishes_the_session() -> None:
|
||||
"""A rebuilt window prompt is bounded by the client's window settings, not
|
||||
by the model: a unit carrying camera frames is hundreds of tokens, and a
|
||||
``context`` window adds the ``previous`` region on top. The replacement
|
||||
branch must therefore check ``replacement_prompt_len`` against
|
||||
``max_model_len - sample_room`` and finish the session with
|
||||
``context_length_exceeded``, exactly like a plain extension."""
|
||||
sched = _make_live_session_scheduler(max_model_len=21)
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids[:] = [0] * 9
|
||||
session.num_prompt_tokens = 9
|
||||
session._output_token_ids[:] = [40]
|
||||
session._all_token_ids.append(40)
|
||||
session.num_computed_tokens = 10
|
||||
session.num_output_placeholders = 0
|
||||
_park_session(sched, session)
|
||||
# The parked append state above is the frame before this update: one
|
||||
# confirmed output token, fully computed.
|
||||
session._output_token_ids[:] = [40]
|
||||
session._all_token_ids[:] = [*session.prompt_token_ids, 40]
|
||||
session.num_computed_tokens = 10
|
||||
session.num_in_flight_tokens = 0
|
||||
sched.num_sampled_tokens_per_step = 1
|
||||
# The replacement plan rebuilds 21 tokens (retained context plus the
|
||||
# append) and one step samples one more, so 21 leaves no room.
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
update.prompt_token_ids = [0] * 20
|
||||
original_prompt = list(session.prompt_token_ids)
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert update.model_intermediate_buffer["duplex"]["stage0_window"]["replacement_prompt_len"] == 21
|
||||
assert session.prompt_token_ids == original_prompt
|
||||
assert session.num_prompt_tokens == 9
|
||||
assert session.status == RequestStatus.FINISHED_ERROR
|
||||
assert session.request_id not in sched.requests
|
||||
assert sched.finished_req_ids == {session.request_id}
|
||||
sched._free_request_blocks.assert_called_once_with(session)
|
||||
client_index, reason = sched._streaming_context_overflow[session.request_id]
|
||||
assert client_index == session.client_index
|
||||
assert reason.startswith("context_length_exceeded: ")
|
||||
assert "21 tokens" in reason and "max_model_len 21" in reason
|
||||
|
||||
engine_core_outputs = _run_idle_step(sched)
|
||||
|
||||
(output,) = engine_core_outputs[session.client_index].outputs
|
||||
assert output.request_id == session.request_id
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
assert output.stop_reason == reason
|
||||
|
||||
|
||||
def test_stage0_window_rebuild_that_leaves_room_to_sample_replaces_the_prompt() -> None:
|
||||
"""One slot above the plan fits: the replacement applies through the normal
|
||||
replacement path."""
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
sched.max_model_len = 25
|
||||
sched.num_sampled_tokens_per_step = 1
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids[:] = session.prompt_token_ids
|
||||
session.num_prompt_tokens = 9
|
||||
session.append_output_token_ids([40])
|
||||
session.num_computed_tokens = 10
|
||||
session.num_output_placeholders = 0
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
update.prompt_token_ids = [0] * 20
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert update.model_intermediate_buffer["meta"]["replace_streaming_prompt"] is True
|
||||
assert update.model_intermediate_buffer["duplex"]["stage0_window"]["replacement_prompt_len"] == 21
|
||||
assert session.prompt_token_ids == [0] * 21
|
||||
assert session.num_prompt_tokens == 21
|
||||
assert session.num_computed_tokens == 0
|
||||
assert session.status == RequestStatus.WAITING
|
||||
sched._free_request_blocks.assert_called_once_with(session)
|
||||
assert not getattr(sched, "_streaming_context_overflow", {})
|
||||
|
||||
|
||||
def test_stage0_window_open_start_zero_is_not_replaced_by_the_context_reserve() -> None:
|
||||
"""A recorded ``open_start`` of 0 is a legitimate empty context prefix. The
|
||||
fallback must be an explicit ``is None`` check: treating 0 as missing
|
||||
substitutes the context reserve, counts the suffix twice, and the worker's
|
||||
rebuild-length check raises on the first replacement."""
|
||||
session = SimpleNamespace(
|
||||
num_prompt_tokens=9,
|
||||
_all_token_ids=[0] * 9 + [40],
|
||||
num_computed_tokens=10,
|
||||
num_output_placeholders=0,
|
||||
_minicpmo45_window_open_start=0,
|
||||
)
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
update.prompt_token_ids = [0] * 21
|
||||
|
||||
assert OmniARScheduler._prepare_minicpmo45_stage0_window(
|
||||
session, update, segment_output_ids=[40], completed_terminator=3
|
||||
)
|
||||
|
||||
# open_start stays 0, so the reported unit spans the whole 12-token
|
||||
# boundary instead of the 9 the context reserve would fold in.
|
||||
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan["dropped_tokens"] == 12
|
||||
assert plan["replacement_prompt_len"] == 19
|
||||
|
||||
|
||||
def test_stage0_window_open_start_falls_back_to_the_context_reserve_when_unset() -> None:
|
||||
session = SimpleNamespace(
|
||||
num_prompt_tokens=9,
|
||||
_all_token_ids=[0] * 9 + [40],
|
||||
num_computed_tokens=10,
|
||||
num_output_placeholders=0,
|
||||
)
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
update.prompt_token_ids = [0] * 21
|
||||
|
||||
assert OmniARScheduler._prepare_minicpmo45_stage0_window(
|
||||
session, update, segment_output_ids=[40], completed_terminator=3
|
||||
)
|
||||
|
||||
# fallback open_start is preserve_len 3 from duplex_first_append_context_tokens,
|
||||
# so the unit is three tokens shorter.
|
||||
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan["dropped_tokens"] == 9
|
||||
assert plan["replacement_prompt_len"] == 22
|
||||
|
||||
|
||||
def test_explicit_streaming_payload_replaces_placeholder_prompt() -> None:
|
||||
sched = _make_scheduler(stage_id=1)
|
||||
sched.chunk_transfer_adapter = SimpleNamespace(
|
||||
|
||||
+108
@@ -60,6 +60,7 @@ def test_custom_cache_dit_enablers_are_registered_explicitly():
|
||||
"Cosmos3OmniPipeline": cd_model_specific.enable_cache_for_cosmos3,
|
||||
"Krea2Pipeline": cd_model_specific.enable_cache_for_krea2,
|
||||
"Magi2Pipeline": cd_model_specific.enable_cache_for_magi2,
|
||||
"MammothModa2DiTPipeline": cd_model_specific.enable_cache_for_mammothmoda2,
|
||||
}
|
||||
|
||||
with patch.dict(cd_backend.CUSTOM_DIT_ENABLERS, {}, clear=True):
|
||||
@@ -116,6 +117,33 @@ def test_magi2_cache_dit_targets_only_nested_repeated_layers(mock_block_adapter,
|
||||
assert get_transformer(pipeline) is transformer_block
|
||||
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.enable_cache_for_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter")
|
||||
def test_mammothmoda2_cache_dit_targets_only_main_layers(mock_block_adapter, mock_enable_cache):
|
||||
pipeline = Mock()
|
||||
transformer = pipeline.gen_transformer
|
||||
layers = torch.nn.ModuleList([torch.nn.Identity()])
|
||||
transformer.layers = layers
|
||||
adapter = mock_block_adapter.return_value
|
||||
refresh = Mock()
|
||||
mock_enable_cache.return_value = refresh
|
||||
|
||||
result = cd_model_specific.enable_cache_for_mammothmoda2(pipeline, SAMPLE_CACHE_CONFIG)
|
||||
|
||||
mock_block_adapter.assert_called_once()
|
||||
adapter_kwargs = mock_block_adapter.call_args.kwargs
|
||||
assert adapter_kwargs["transformer"] is transformer
|
||||
assert adapter_kwargs["blocks"] == [layers]
|
||||
assert adapter_kwargs["has_separate_cfg"] is True
|
||||
assert adapter_kwargs["check_forward_pattern"] is True
|
||||
assert result.refresh is refresh
|
||||
assert result.targets == (adapter,)
|
||||
get_transformer = mock_enable_cache.call_args.kwargs["get_pipeline_transformer"]
|
||||
assert get_transformer(pipeline) is transformer
|
||||
# Sequential-CFG parity is mandatory while cache-dit is installed.
|
||||
assert pipeline._cache_dit_requires_paired_cfg is True
|
||||
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit.summary")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter.is_cached", return_value=True)
|
||||
def test_cache_summary_uses_custom_nested_targets(mock_is_cached, mock_summary):
|
||||
@@ -338,3 +366,83 @@ def test_summary_with_no_transformer_is_nonfatal():
|
||||
pass
|
||||
|
||||
cache_summary(pipeline=FakePipeline())
|
||||
|
||||
|
||||
# This test is skipped on ROCm since rocm_unquantized_gemm doesn't support CPU backend
|
||||
@pytest.mark.skipif(
|
||||
current_omni_platform.is_rocm(),
|
||||
reason="vLLM ROCm custom ops lack CPU fallback",
|
||||
)
|
||||
def test_mammothmoda2_cache_dit_runs_end_to_end_on_tiny_model(request: pytest.FixtureRequest):
|
||||
"""A tiny MammothModa2 DiT runs through Cache-DiT with paired CFG forwards.
|
||||
|
||||
Exercises the Pattern_3 contract end to end: after enabling the custom
|
||||
enabler, blocks must be callable as ``block(hidden_states, **kwargs)``
|
||||
and the pipeline's two-forwards-per-step CFG cadence must drive the
|
||||
cache context without shape or parity errors.
|
||||
"""
|
||||
from cache_dit import BlockAdapter
|
||||
|
||||
from vllm_omni.diffusion.models.mammoth_moda2.mammothmoda2_dit_model import Transformer2DModel
|
||||
from vllm_omni.diffusion.models.mammoth_moda2.rope_real import RotaryPosEmbedReal
|
||||
|
||||
# MammothModa2 attention resolves through the shared Omni attention layer,
|
||||
# which picks the platform-default backend at construction time. Pin
|
||||
# TORCH_SDPA so a CUDA-visible host does not select FA3 for CPU tensors.
|
||||
with _force_torch_sdpa():
|
||||
model = Transformer2DModel(
|
||||
patch_size=2,
|
||||
in_channels=4,
|
||||
hidden_size=96,
|
||||
num_layers=4,
|
||||
num_refiner_layers=1,
|
||||
num_attention_heads=2,
|
||||
num_kv_heads=2,
|
||||
multiple_of=8,
|
||||
axes_dim_rope=(16, 16, 16),
|
||||
axes_lens=(300, 512, 512),
|
||||
text_feat_dim=16,
|
||||
)
|
||||
model.eval()
|
||||
|
||||
MammothModa2Pipeline = type("MammothModa2DiTPipeline", (), {})
|
||||
pipeline = MammothModa2Pipeline()
|
||||
pipeline.gen_transformer = model
|
||||
backend = CacheDiTBackend(DiffusionCacheConfig())
|
||||
backend.enable(pipeline)
|
||||
request.addfinalizer(lambda: backend.disable(pipeline))
|
||||
assert BlockAdapter.is_cached(model)
|
||||
|
||||
num_inference_steps = 8
|
||||
backend.refresh(pipeline, num_inference_steps=num_inference_steps)
|
||||
|
||||
freqs_cis = RotaryPosEmbedReal.get_freqs_real((16, 16, 16), (300, 512, 512), theta=10000)
|
||||
latents = torch.randn(1, 4, 8, 8)
|
||||
text = torch.randn(1, 5, 16)
|
||||
negative = torch.randn(1, 3, 16)
|
||||
text_mask = torch.ones(1, 5, dtype=torch.bool)
|
||||
negative_mask = torch.ones(1, 3, dtype=torch.bool)
|
||||
|
||||
with torch.no_grad():
|
||||
for step in range(num_inference_steps):
|
||||
timestep = torch.full((1,), 1.0 - step / num_inference_steps)
|
||||
# Conditional pass ...
|
||||
cond = model(
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_hidden_states=text,
|
||||
freqs_cis=freqs_cis,
|
||||
text_attention_mask=text_mask,
|
||||
)
|
||||
# ... always followed by the unconditional pass (parity).
|
||||
uncond = model(
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_hidden_states=negative,
|
||||
freqs_cis=freqs_cis,
|
||||
text_attention_mask=negative_mask,
|
||||
)
|
||||
latents = uncond + 4.0 * (cond - uncond)
|
||||
|
||||
assert latents.shape == (1, 4, 8, 8)
|
||||
assert torch.isfinite(latents).all()
|
||||
@@ -1,13 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import cache_dit
|
||||
import pytest
|
||||
import torch
|
||||
from cache_dit import BlockAdapter
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig, TransformerConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTBackend, RequestScopedCacheDiTRuntime
|
||||
from vllm_omni.diffusion.cache.selector import get_cache_backend
|
||||
from vllm_omni.diffusion.config import set_current_diffusion_config
|
||||
from vllm_omni.diffusion.data import (
|
||||
AttentionConfig,
|
||||
DiffusionCacheConfig,
|
||||
DiffusionOutput,
|
||||
OmniDiffusionConfig,
|
||||
TransformerConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.models.interface import adopt_request_scoped_cache_dit
|
||||
from vllm_omni.diffusion.models.mammoth_moda2 import pipeline_mammothmoda2_dit
|
||||
from vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit import (
|
||||
MammothModa2DiTPipeline,
|
||||
@@ -17,6 +31,7 @@ from vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit import (
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -110,6 +125,10 @@ def _pipeline_shell() -> MammothModa2DiTPipeline:
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.config = _build_mammoth_config(_od_config())
|
||||
pipeline._llm_hidden_size = 8
|
||||
# Cache-DiT runtime state that __init__ normally creates (disabled here,
|
||||
# matching a pipeline built without a cache_backend).
|
||||
pipeline._cache_dit_runtime = RequestScopedCacheDiTRuntime(pipeline)
|
||||
pipeline._cache_dit_config = None
|
||||
return pipeline
|
||||
|
||||
|
||||
@@ -118,12 +137,14 @@ def _batch(
|
||||
request_id: str = "req-7",
|
||||
prompt: object | None = None,
|
||||
sampling: OmniDiffusionSamplingParams | None = None,
|
||||
height: int = 32,
|
||||
width: int = 48,
|
||||
) -> DiffusionRequestBatch:
|
||||
if prompt is None:
|
||||
prompt = {
|
||||
"prompt": "",
|
||||
"height": 32,
|
||||
"width": 48,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"additional_information": {
|
||||
"full_hidden_states": torch.arange(32, dtype=torch.float32).reshape(4, 8),
|
||||
"full_token_ids": [10, 11, 100, 101],
|
||||
@@ -390,3 +411,162 @@ def test_forward_rejects_missing_visual_tokens_before_model_access() -> None:
|
||||
}
|
||||
with pytest.raises(ValueError, match="no visual-token hidden states.*req-empty"):
|
||||
_pipeline_shell().forward(_batch(request_id="req-empty", prompt=prompt))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _force_torch_sdpa():
|
||||
"""Pin TORCH_SDPA so CPU shape tests do not pick CUDA-only backends (FA3)."""
|
||||
od_config = SimpleNamespace(
|
||||
diffusion_attention_config=AttentionConfig(default="TORCH_SDPA"),
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
with set_current_diffusion_config(od_config):
|
||||
yield
|
||||
|
||||
|
||||
def _constructible_raw_config() -> dict:
|
||||
"""Checkpoint config whose DiT/VAE are small enough to actually run on CPU."""
|
||||
return {
|
||||
"model_type": "mammothmoda2",
|
||||
"llm_config": {
|
||||
"model_type": "mammothmoda2_qwen2_5_vl",
|
||||
"text_config": {
|
||||
"model_type": "mammothmoda2_qwen2_5_vl_text",
|
||||
"hidden_size": 8,
|
||||
"gen_vocab_start_index": 100,
|
||||
},
|
||||
},
|
||||
"gen_vae_config": {"block_out_channels": [8, 8], "norm_num_groups": 8},
|
||||
"gen_dit_config": {
|
||||
"patch_size": 2,
|
||||
"in_channels": 4,
|
||||
"hidden_size": 96,
|
||||
"num_layers": 4,
|
||||
"num_refiner_layers": 1,
|
||||
"num_attention_heads": 2,
|
||||
"num_kv_heads": 2,
|
||||
"multiple_of": 8,
|
||||
"axes_dim_rope": [16, 16, 16],
|
||||
"axes_lens": [300, 512, 512],
|
||||
"text_feat_dim": 16,
|
||||
},
|
||||
# Top-level axes feed the pipeline-owned rotary table (see pipeline __init__).
|
||||
"gen_axes_dim_rope": [16, 16, 16],
|
||||
"gen_axes_lens": [300, 512, 512],
|
||||
}
|
||||
|
||||
|
||||
def _cache_dit_od_config() -> OmniDiffusionConfig:
|
||||
return OmniDiffusionConfig(
|
||||
model="/models/MammothModa2-Preview",
|
||||
model_class_name="MammothModa2DiTPipeline",
|
||||
tf_model_config=TransformerConfig.from_dict(_constructible_raw_config()),
|
||||
cache_backend="cache_dit",
|
||||
cache_config=DiffusionCacheConfig(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_omni_platform.is_rocm(),
|
||||
reason="vLLM ROCm custom ops lack CPU fallback",
|
||||
)
|
||||
def test_forward_transitions_request_scoped_cache_dit_across_requests(request: pytest.FixtureRequest) -> None:
|
||||
"""Cache-DiT request lifecycle on one production-built pipeline instance.
|
||||
|
||||
Mirrors the diffusion runner startup (``get_cache_backend`` → ``enable`` →
|
||||
``adopt_request_scoped_cache_dit``) on a tiny pipeline built through the
|
||||
real ``__init__``, then drives three requests through the real ``forward()``:
|
||||
|
||||
1. a partial-CFG request installs the request-scoped cache context:
|
||||
cond/uncond forwards stay paired on every denoise step (parity
|
||||
accounting), even outside ``cfg_range``, and blocks record cached steps;
|
||||
2. a guidance=1.0 request runs with hooks disabled (``prepare(None)``) and
|
||||
its output is bit-identical to a reference captured before the backend
|
||||
was ever enabled, proving a clean uninstall with no stale module state;
|
||||
3. a CFG request with a different step count and image size re-installs the
|
||||
hooks through a fresh context, so no stale shape or step count survives.
|
||||
"""
|
||||
torch.manual_seed(0) # Deterministic random init → deterministic cache hits.
|
||||
with _force_torch_sdpa():
|
||||
pipeline = MammothModa2DiTPipeline(od_config=_cache_dit_od_config())
|
||||
pipeline.eval()
|
||||
assert pipeline._cache_dit_config is not None
|
||||
request.addfinalizer(pipeline._cache_dit_runtime.disable)
|
||||
|
||||
# The default prompt carries 2 text + 2 visual condition tokens, so the
|
||||
# conditional pass sees 4 tokens and the uncond pass (empty prompt) sees 0.
|
||||
cond_tokens, uncond_tokens = 4, 0
|
||||
seq_lens: list[int] = []
|
||||
|
||||
def record_call(_module, _args, kwargs):
|
||||
text = kwargs.get("text_hidden_states")
|
||||
seq_lens.append(int(text.shape[1]) if text is not None else -1)
|
||||
|
||||
hook = pipeline.gen_transformer.register_forward_pre_hook(record_call, with_kwargs=True)
|
||||
|
||||
unguided_sampling = OmniDiffusionSamplingParams(
|
||||
height=32, width=48, seed=4321, guidance_scale=1.0, num_inference_steps=8
|
||||
)
|
||||
partial_cfg_sampling = OmniDiffusionSamplingParams(
|
||||
height=32,
|
||||
width=48,
|
||||
seed=1234,
|
||||
guidance_scale=4.0,
|
||||
num_inference_steps=8,
|
||||
extra_args={"cfg_range": [0.2, 0.8]},
|
||||
)
|
||||
changed_cfg_sampling = OmniDiffusionSamplingParams(
|
||||
height=48,
|
||||
width=64,
|
||||
seed=777,
|
||||
guidance_scale=6.0,
|
||||
num_inference_steps=10,
|
||||
extra_args={"cfg_range": [0.1, 0.9]},
|
||||
)
|
||||
|
||||
# Reference run for phase 2: identical request while hooks were never
|
||||
# installed on this pipeline.
|
||||
assert not pipeline.is_cache_dit_enabled()
|
||||
reference = pipeline.forward(_batch(sampling=unguided_sampling)).output
|
||||
assert seq_lens == [cond_tokens] * 8
|
||||
assert torch.isfinite(reference.float()).all()
|
||||
|
||||
# Runner startup: enable once, then the pipeline owns all transitions.
|
||||
backend = get_cache_backend("cache_dit", DiffusionCacheConfig())
|
||||
assert isinstance(backend, CacheDiTBackend)
|
||||
backend.enable(pipeline)
|
||||
assert adopt_request_scoped_cache_dit(pipeline, backend)
|
||||
assert pipeline.is_cache_dit_enabled()
|
||||
|
||||
# Phase 1: partial CFG keeps cond/uncond paired on every step, including
|
||||
# the steps whose normalized index falls outside cfg_range [0.2, 0.8].
|
||||
seq_lens.clear()
|
||||
pipeline.forward(_batch(sampling=partial_cfg_sampling))
|
||||
assert seq_lens == [cond_tokens, uncond_tokens] * 8
|
||||
assert pipeline.is_cache_dit_enabled()
|
||||
assert BlockAdapter.is_cached(pipeline.gen_transformer)
|
||||
hits = cache_dit.summary(pipeline.gen_transformer, logging=False)[0]
|
||||
assert hits.cached_steps or hits.cfg_cached_steps
|
||||
|
||||
# Phase 2: no-CFG request disables the hooks and must reproduce the
|
||||
# never-enabled reference bit-for-bit.
|
||||
seq_lens.clear()
|
||||
unguided = pipeline.forward(_batch(sampling=unguided_sampling)).output
|
||||
assert seq_lens == [cond_tokens] * 8
|
||||
assert not pipeline.is_cache_dit_enabled()
|
||||
assert not BlockAdapter.is_cached(pipeline.gen_transformer)
|
||||
assert torch.equal(unguided, reference)
|
||||
|
||||
# Phase 3: CFG with a new step count and image size re-installs the hooks
|
||||
# on a fresh context; output shape follows the new request geometry.
|
||||
seq_lens.clear()
|
||||
regenerated = pipeline.forward(_batch(sampling=changed_cfg_sampling, height=48, width=64)).output
|
||||
assert seq_lens == [cond_tokens, uncond_tokens] * 10
|
||||
assert pipeline.is_cache_dit_enabled()
|
||||
assert BlockAdapter.is_cached(pipeline.gen_transformer)
|
||||
hits = cache_dit.summary(pipeline.gen_transformer, logging=False)[0]
|
||||
assert hits.cached_steps or hits.cfg_cached_steps
|
||||
assert regenerated.shape == (1, 3, 12, 16)
|
||||
assert torch.isfinite(regenerated.float()).all()
|
||||
|
||||
hook.remove()
|
||||
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Drive complete native duplex turns while Stage-0 repeatedly rebuilds KV."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.clients.duplex import DuplexClient, EventCollector, read_pcm16_wav, wait_for_condition
|
||||
from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
|
||||
|
||||
|
||||
async def run_window_turn(
|
||||
*,
|
||||
url: str,
|
||||
model: str,
|
||||
input_wav: Path,
|
||||
mode: str,
|
||||
ref_audio: Path | None = None,
|
||||
repeats: int = 2,
|
||||
video_frames: list[str] | None = None,
|
||||
buffered_flush: bool = False,
|
||||
timeout_s: float = 240.0,
|
||||
) -> dict[str, object]:
|
||||
"""Use aggressive limits to exercise multiple windows in one audio turn."""
|
||||
pcm = read_pcm16_wav(input_wav) * repeats
|
||||
collector = EventCollector()
|
||||
reference = (
|
||||
"data:audio/wav;base64," + base64.b64encode(ref_audio.read_bytes()).decode("ascii")
|
||||
if ref_audio is not None
|
||||
else None
|
||||
)
|
||||
config = create_duplex_session_config(
|
||||
modalities=("text", "audio") if ref_audio is not None else ("text",),
|
||||
ref_audio=reference,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"sliding_window_mode": mode,
|
||||
"context_max_units": 1,
|
||||
"context_previous_max_tokens": 16,
|
||||
"basic_window_high_tokens": 64,
|
||||
"basic_window_low_tokens": 32,
|
||||
},
|
||||
)
|
||||
client = DuplexClient(
|
||||
url, model=model, config=config, reconnect=None, heartbeat_interval_s=None, handshake_timeout_s=timeout_s
|
||||
)
|
||||
async with client:
|
||||
reader = asyncio.create_task(collector.consume(client))
|
||||
collector.add({"type": "session.created", "session": client.session_info})
|
||||
try:
|
||||
frames_sent = await client.stream_pcm(
|
||||
pcm,
|
||||
chunk_ms=2500 if buffered_flush else 200,
|
||||
realtime=not buffered_flush,
|
||||
is_speech=True,
|
||||
video_frames=video_frames,
|
||||
)
|
||||
await client.commit(final=True)
|
||||
await wait_for_condition(
|
||||
lambda: collector.count("input_audio_buffer.committed") > 0 or bool(collector.errors()),
|
||||
timeout_s=timeout_s,
|
||||
label="final input commit",
|
||||
)
|
||||
await wait_for_condition(
|
||||
lambda: collector.count("response.done") > 0 or bool(collector.errors()),
|
||||
timeout_s=timeout_s,
|
||||
label="completed windowed response",
|
||||
)
|
||||
# Final execution follows the commit acknowledgement asynchronously.
|
||||
await asyncio.sleep(3.0)
|
||||
await client.close(timeout_s=timeout_s)
|
||||
await asyncio.wait_for(reader, timeout=5.0)
|
||||
finally:
|
||||
if not reader.done():
|
||||
reader.cancel()
|
||||
await asyncio.gather(reader, return_exceptions=True)
|
||||
return {
|
||||
"created": collector.count("session.created"),
|
||||
"committed": collector.count("input_audio_buffer.committed"),
|
||||
"done": collector.count("response.done"),
|
||||
"closed": collector.count("session.closed"),
|
||||
"errors": collector.errors(),
|
||||
"audio_bytes": len(collector.audio_bytes()),
|
||||
"frames_sent": frames_sent,
|
||||
"input_seconds": len(pcm) / 32000,
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Real-model regression for the scheduler-owned Thinker sliding window."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||
SERVER_PARAMS,
|
||||
duplex_camera_frames,
|
||||
realtime_url,
|
||||
resolve_ref_audio,
|
||||
validated_input_wav,
|
||||
)
|
||||
from tests.e2e.online_serving.helpers.minicpmo_window_e2e import run_window_turn
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
pytestmark = [pytest.mark.omni, pytest.mark.advanced_model]
|
||||
|
||||
|
||||
def _assert_complete(result, *, require_audio=False):
|
||||
assert result["created"] == 1
|
||||
assert result["committed"] > 0
|
||||
assert result["done"] > 0
|
||||
assert result["closed"] == 1
|
||||
assert result["errors"] == []
|
||||
if require_audio:
|
||||
assert result["audio_bytes"] > 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode,reference", [("basic", False), ("context", False), ("context", True)])
|
||||
def test_window_rebuild_and_next_session(omni_server, mode, reference):
|
||||
# Two independent sessions exercise admission after final-unit execution.
|
||||
for _ in range(2):
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
ref_audio=resolve_ref_audio() if reference else None,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=reference)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode,camera", [("basic", False), ("context", False), ("context", True)])
|
||||
def test_window_continuous_input(omni_server, tmp_path, mode, camera):
|
||||
# 24 repeats cover over two minutes and many canonical KV replacements.
|
||||
frames = duplex_camera_frames(seconds=4, cache_dir=tmp_path / "camera") if camera else None
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
repeats=24,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
video_frames=frames,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=True)
|
||||
assert isinstance(result["input_seconds"], (int, float)) and result["input_seconds"] > 120
|
||||
if camera:
|
||||
assert isinstance(result["frames_sent"], int) and result["frames_sent"] >= 120
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode", ["off", "basic", "context"])
|
||||
def test_window_buffered_flush(omni_server, mode):
|
||||
# Burst appends cross processor chunk boundaries and leave a final tail.
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
buffered_flush=True,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=True)
|
||||
@@ -0,0 +1,117 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.config import DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.contracts import DuplexFence
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex import plugin as module
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _plugin():
|
||||
return module.MiniCPMO45DuplexPlugin(lambda *args: None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_window_configuration_is_prepared_by_plugin(monkeypatch):
|
||||
tokenizer = SimpleNamespace(
|
||||
all_special_ids=[90],
|
||||
unk_token_id=-1,
|
||||
eos_token_id=99,
|
||||
encode=lambda text, add_special_tokens=False: [ord(char) for char in text],
|
||||
convert_tokens_to_ids=lambda token: {"<|listen|>": 91}.get(token, -1),
|
||||
)
|
||||
monkeypatch.setattr(module, "_load_tokenizer", lambda config: tokenizer)
|
||||
config = DuplexSessionConfig(
|
||||
modalities=("text",),
|
||||
extra_body={
|
||||
"sliding_window_mode": "basic",
|
||||
"basic_window_high_tokens": 120,
|
||||
"basic_window_low_tokens": 80,
|
||||
},
|
||||
)
|
||||
runtime = await _plugin().prepare_runtime_config(config, model_config=None)
|
||||
assert runtime["duplex_window_config"]["sliding_window_mode"] == "basic"
|
||||
assert "sliding_window_mode" not in config.extra_body
|
||||
assert runtime["duplex_window_prefix_tokens"] > 0
|
||||
assert runtime["duplex_window_suffix_token_ids"]
|
||||
assert runtime["duplex_window_previous_marker_token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_window_configuration_is_rejected():
|
||||
config = DuplexSessionConfig(
|
||||
modalities=("text",),
|
||||
extra_body={
|
||||
"basic_window_high_tokens": 80,
|
||||
"basic_window_low_tokens": 80,
|
||||
},
|
||||
)
|
||||
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError) as exc:
|
||||
await _plugin().prepare_runtime_config(config, model_config=None)
|
||||
assert exc.value.code == "invalid_sliding_window_config"
|
||||
|
||||
|
||||
def test_window_update_cannot_change_created_configuration():
|
||||
config = DuplexSessionConfig(extra_body={"sliding_window_mode": "basic"})
|
||||
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError) as exc:
|
||||
_plugin().runtime_config_for_update(config, {"duplex_window_config": {"sliding_window_mode": "off"}})
|
||||
assert exc.value.code == "sliding_window_update_unsupported"
|
||||
|
||||
|
||||
def test_window_internal_configuration_is_server_owned():
|
||||
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError):
|
||||
_plugin().validate_client_extra_body({"duplex_window_config": {}})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("final", [False, True])
|
||||
@pytest.mark.parametrize("seq,samples,expected", [(1, 32000, 16), (1, 48000, 28), (2, 16000, 13)])
|
||||
def test_first_and_final_append_reserve_exact_window_input(seq, samples, expected, final):
|
||||
prompt = module.build_duplex_data_plane_prompt(
|
||||
request_id="window-request",
|
||||
fence=DuplexFence("sid", turn_id=1),
|
||||
session_config={},
|
||||
runtime_config={"duplex_first_append_context_tokens": 5},
|
||||
seq=seq,
|
||||
turn_seq=seq,
|
||||
payload={
|
||||
"audio": base64.b64encode(bytes(samples * 4)).decode(),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
},
|
||||
final=final,
|
||||
)
|
||||
assert len(prompt["prompt_token_ids"]) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq", [1, 2, 3])
|
||||
def test_final_exact_chunk_append_reserves_one_unit_not_two(seq):
|
||||
"""A final append whose audio is already an exact number of chunks must
|
||||
reserve exactly what Stage0 feeds: one unit per chunk (``<unit>`` plus its
|
||||
audio) and the closure pair for every unit after the first. Serving pads
|
||||
the final residual itself, so the old extra 12-slot silent-unit
|
||||
reservation only put 12 pad positions ahead of the audio in the KV."""
|
||||
|
||||
def _budget(*, seq, final):
|
||||
prompt = module.build_duplex_data_plane_prompt(
|
||||
request_id="final-request",
|
||||
fence=DuplexFence("sid", turn_id=1),
|
||||
session_config={},
|
||||
runtime_config={"duplex_first_append_context_tokens": 0},
|
||||
seq=seq,
|
||||
turn_seq=seq,
|
||||
payload={
|
||||
"audio": base64.b64encode(bytes(16000 * 4)).decode(),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
},
|
||||
final=final,
|
||||
)
|
||||
return len(prompt["prompt_token_ids"])
|
||||
|
||||
assert _budget(seq=seq, final=True) == _budget(seq=seq, final=False)
|
||||
@@ -51,6 +51,7 @@ from vllm_omni.engine.duplex.plugin import (
|
||||
PcmAppendReservation,
|
||||
)
|
||||
from vllm_omni.engine.duplex.session import manager as session_manager_module
|
||||
from vllm_omni.engine.duplex.session.context import DuplexSessionTasks
|
||||
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession
|
||||
from vllm_omni.engine.duplex.session.lease import DuplexLeaseActivity
|
||||
from vllm_omni.engine.duplex.session.manager import DuplexSessionManager
|
||||
@@ -1612,3 +1613,34 @@ async def test_reaper_loop_survives_one_cleanup_failure(first_cleanup_delay: flo
|
||||
finally:
|
||||
shutdown.set()
|
||||
await asyncio.wait_for(task, timeout=5.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cancellation waits #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
async def test_cancel_append_tasks_absorbs_a_task_that_outlives_the_wait() -> None:
|
||||
"""The wait after cancelling is allowed to time out; that is not an error.
|
||||
|
||||
On Python 3.10 the timeout arrives as ``asyncio.TimeoutError``, which is a
|
||||
different class from the builtin ``TimeoutError`` until 3.11.
|
||||
"""
|
||||
|
||||
async def outlives_the_first_cancel() -> bool:
|
||||
try:
|
||||
await asyncio.sleep(5)
|
||||
except asyncio.CancelledError:
|
||||
await asyncio.sleep(1)
|
||||
raise
|
||||
return True
|
||||
|
||||
tasks = DuplexSessionTasks()
|
||||
task = asyncio.create_task(outlives_the_first_cancel())
|
||||
await asyncio.sleep(0)
|
||||
tasks.track_append_task(task, epoch=0, final=False, response_bound=False)
|
||||
tasks.append_tail = task
|
||||
|
||||
assert await tasks.cancel_append_tasks(timeout_s=0.05) is True
|
||||
assert tasks.append_tail is None
|
||||
await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), timeout=5.0)
|
||||
@@ -771,6 +771,41 @@ async def test_cancel_response_aborts_the_stage_request_and_reports_playback() -
|
||||
await close_harness(h)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_response_absorbs_a_response_task_that_outlives_the_wait() -> None:
|
||||
"""The 0.25 s wait after cancelling the response task may time out.
|
||||
|
||||
On Python 3.10 that timeout is ``asyncio.TimeoutError``, not the builtin
|
||||
``TimeoutError``, and it must not stop the cancel from completing.
|
||||
"""
|
||||
|
||||
async def outlives_the_first_cancel() -> None:
|
||||
try:
|
||||
await asyncio.sleep(5)
|
||||
except asyncio.CancelledError:
|
||||
await asyncio.sleep(1)
|
||||
raise
|
||||
|
||||
h = await open_harness()
|
||||
task = asyncio.create_task(outlives_the_first_cancel())
|
||||
try:
|
||||
await h.run(append_audio())
|
||||
request_id = h.stage0_request_id()
|
||||
await h.deliver_and_settle(tts_output(request_id, samples=24000, text="hello"))
|
||||
h.runner.tasks.active_response_task = task
|
||||
|
||||
events = await h.run(commands.CancelResponse())
|
||||
# The runner is still inside that wait when the mailbox goes quiet.
|
||||
events += await h.settle(idle_s=0.6, timeout_s=5.0)
|
||||
assert find(events, "response.done").status == "cancelled"
|
||||
assert "error" not in types(events)
|
||||
assert h.session.epoch == 1
|
||||
finally:
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
await close_harness(h)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_listen_decision_is_consumed_and_never_forwarded_to_tts() -> None:
|
||||
h = await open_harness()
|
||||
|
||||
@@ -505,6 +505,75 @@ class TestSpeechAPI:
|
||||
assert voice_info["file_size"] == len(audio_content)
|
||||
response = client.delete("/v1/audio/voices/test_voice")
|
||||
|
||||
def test_upload_voice_rejects_builtin_name(self, client):
|
||||
"""A name colliding with a built-in or precomputed voice is rejected."""
|
||||
handler = client.app.state.openai_serving_speech
|
||||
handler._adapter.capabilities.supported_speakers = {"vivian"}
|
||||
handler._adapter.capabilities.precomputed_speakers = {"ono_anna": {}}
|
||||
|
||||
files = {"audio_sample": ("test.wav", b"fake audio content" * 1000, "audio/wav")}
|
||||
for reserved in ("Vivian", "ono_anna"):
|
||||
response = client.post("/v1/audio/voices", files=files, data={"consent": "c1", "name": reserved})
|
||||
assert response.status_code == 400
|
||||
assert "reserved" in response.json()["detail"]
|
||||
assert not handler.uploaded_speakers
|
||||
|
||||
def test_upload_voice_overwrites_same_name_by_default(self, client):
|
||||
"""Default policy: re-registering an uploaded name replaces it in place."""
|
||||
files = {"audio_sample": ("test.wav", b"first take", "audio/wav")}
|
||||
data = {"consent": "c1", "name": "test_voice_ow"}
|
||||
assert client.post("/v1/audio/voices", files=files, data=data).status_code == 200
|
||||
|
||||
files = {"audio_sample": ("test.wav", b"second take, longer content", "audio/wav")}
|
||||
response = client.post("/v1/audio/voices", files=files, data=data)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["voice"]["file_size"] == len(b"second take, longer content")
|
||||
client.delete("/v1/audio/voices/test_voice_ow")
|
||||
|
||||
def test_upload_voice_immutable_policy_requires_delete(self, client):
|
||||
"""VLLM_OMNI_SPEAKER_REGISTRATION_POLICY=immutable rejects duplicates until deleted."""
|
||||
handler = client.app.state.openai_serving_speech
|
||||
handler._registration_policy = "immutable"
|
||||
try:
|
||||
files = {"audio_sample": ("test.wav", b"fake audio content" * 1000, "audio/wav")}
|
||||
data = {"consent": "c1", "name": "test_voice_imm"}
|
||||
assert client.post("/v1/audio/voices", files=files, data=data).status_code == 200
|
||||
|
||||
response = client.post("/v1/audio/voices", files=files, data=data)
|
||||
assert response.status_code == 400
|
||||
assert "immutable" in response.json()["detail"]
|
||||
|
||||
assert client.delete("/v1/audio/voices/test_voice_imm").status_code == 200
|
||||
assert client.post("/v1/audio/voices", files=files, data=data).status_code == 200
|
||||
finally:
|
||||
handler._registration_policy = "overwrite"
|
||||
client.delete("/v1/audio/voices/test_voice_imm")
|
||||
|
||||
def test_invalid_registration_policy_fails_startup(self, mocker, monkeypatch, tmp_path):
|
||||
"""An unknown VLLM_OMNI_SPEAKER_REGISTRATION_POLICY is a configuration error, not a silent fallback."""
|
||||
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("VLLM_OMNI_SPEAKER_REGISTRATION_POLICY", "append")
|
||||
engine_client = mocker.MagicMock()
|
||||
engine_client.default_sampling_params_list = [{}]
|
||||
with pytest.raises(ValueError, match="VLLM_OMNI_SPEAKER_REGISTRATION_POLICY"):
|
||||
OmniOpenAIServingSpeech(
|
||||
engine_client=engine_client, models=mocker.MagicMock(), request_logger=mocker.MagicMock()
|
||||
)
|
||||
|
||||
def test_restored_upload_shadowing_builtin_is_dropped(self, client):
|
||||
"""A pre-guard upload restored under a built-in name must not keep shadowing it."""
|
||||
handler = client.app.state.openai_serving_speech
|
||||
handler._adapter.capabilities.supported_speakers = {"vivian"}
|
||||
handler._adapter.capabilities.precomputed_speakers = {}
|
||||
handler.uploaded_speakers["vivian"] = {"name": "vivian", "file_path": "/tmp/vivian.safetensors"}
|
||||
handler.uploaded_speakers["keep_me"] = {"name": "keep_me", "file_path": "/tmp/keep_me.safetensors"}
|
||||
|
||||
handler._drop_shadowing_uploads()
|
||||
|
||||
assert "vivian" not in handler.uploaded_speakers
|
||||
assert "keep_me" in handler.uploaded_speakers
|
||||
handler.uploaded_speakers.pop("keep_me")
|
||||
|
||||
def test_upload_voice_with_ref_text(self, client, tmp_path):
|
||||
"""Test voice upload with ref_text enables in-context cloning."""
|
||||
audio_content = b"fake audio content" * 1000
|
||||
|
||||
@@ -33,9 +33,9 @@ def _make_omni(*, stage_types: list[str]) -> AsyncOmni:
|
||||
omni._final_output_handler = lambda: None
|
||||
omni.reset_mm_cache = AsyncMock()
|
||||
|
||||
stage_clients = [SimpleNamespace(stage_type=stage_type) for stage_type in stage_types]
|
||||
stage_configs = [SimpleNamespace(stage_type=stage_type) for stage_type in stage_types]
|
||||
omni.engine = SimpleNamespace(
|
||||
stage_clients=stage_clients,
|
||||
stage_configs=stage_configs,
|
||||
stage_vllm_configs=[None] * len(stage_types),
|
||||
collective_rpc_async=AsyncMock(return_value=[True]),
|
||||
)
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||
from vllm_omni.config.stage_config import StageConfig
|
||||
from vllm_omni.config.stage_config import StageConfig, StageType
|
||||
from vllm_omni.engine.async_omni_engine import StageRuntimeInfo
|
||||
from vllm_omni.engine.messages import ErrorMessage, OutputMessage
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
@@ -107,7 +107,13 @@ class FakeAsyncOmniEngine:
|
||||
self.stage_metadata = stage_metadata or [THREE_STAGE_META[-1]]
|
||||
self.num_stages = len(self.stage_metadata)
|
||||
self.stage_configs = [
|
||||
StageConfig(stage_id=i, model_stage="dummy-model").to_omegaconf() for i in range(self.num_stages)
|
||||
StageConfig(
|
||||
stage_id=i,
|
||||
model_stage="dummy-model",
|
||||
stage_type=StageType(metadata.stage_type),
|
||||
is_comprehension=getattr(metadata, "is_comprehension", False),
|
||||
).to_omegaconf()
|
||||
for i, metadata in enumerate(self.stage_metadata)
|
||||
]
|
||||
self.default_sampling_params_list = default_sampling_params_list or [
|
||||
SamplingParams(max_tokens=8) for _ in range(self.num_stages)
|
||||
@@ -481,7 +487,7 @@ def test_model_config_and_vllm_config_forward_from_comprehension_stage():
|
||||
io_processor = SimpleNamespace(name="io-processor")
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = SimpleNamespace(
|
||||
stage_clients=[SimpleNamespace(is_comprehension=False), SimpleNamespace(is_comprehension=True)],
|
||||
stage_configs=[SimpleNamespace(is_comprehension=False), SimpleNamespace(is_comprehension=True)],
|
||||
stage_vllm_configs=[None, vllm_config],
|
||||
)
|
||||
omni.input_processor = input_processor
|
||||
@@ -502,7 +508,7 @@ def test_openai_serving_models_can_consume_async_omni_compat_attrs():
|
||||
io_processor = SimpleNamespace(name="io-processor")
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = SimpleNamespace(
|
||||
stage_clients=[SimpleNamespace(is_comprehension=True)],
|
||||
stage_configs=[SimpleNamespace(is_comprehension=True)],
|
||||
stage_vllm_configs=[vllm_config],
|
||||
)
|
||||
omni.input_processor = input_processor
|
||||
@@ -524,10 +530,11 @@ def test_get_diffusion_od_config_returns_diffusion_stage_config():
|
||||
diffusion_od_config = object()
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = SimpleNamespace(
|
||||
stage_configs=[SimpleNamespace(stage_type="llm"), SimpleNamespace(stage_type="diffusion")],
|
||||
stage_clients=[
|
||||
SimpleNamespace(stage_type="llm"),
|
||||
SimpleNamespace(stage_type="diffusion", od_config=diffusion_od_config),
|
||||
]
|
||||
],
|
||||
)
|
||||
|
||||
assert omni.get_diffusion_od_config() is diffusion_od_config
|
||||
@@ -537,15 +544,33 @@ def test_get_diffusion_od_config_falls_back_to_inner_engine():
|
||||
diffusion_od_config = object()
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni.engine = SimpleNamespace(
|
||||
stage_configs=[SimpleNamespace(stage_type="llm"), SimpleNamespace(stage_type="diffusion")],
|
||||
stage_clients=[
|
||||
SimpleNamespace(stage_type="llm"),
|
||||
SimpleNamespace(stage_type="diffusion", _engine=SimpleNamespace(od_config=diffusion_od_config)),
|
||||
]
|
||||
],
|
||||
)
|
||||
|
||||
assert omni.get_diffusion_od_config() is diffusion_od_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_rejects_diffusion_list_prompt_without_stage_clients(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
engine = FakeAsyncOmniEngine(stage_metadata=DIFFUSION_ONLY_META)
|
||||
engine.stage_clients = []
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = AsyncOmni("dummy-model")
|
||||
try:
|
||||
with pytest.raises(ValueError, match="single prompt per request"):
|
||||
async for _ in app.generate(prompt=["prompt-1", "prompt-2"], request_id="req-1"):
|
||||
pass
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_yields_only_final_stage_outputs(monkeypatch: pytest.MonkeyPatch):
|
||||
engine = FakeAsyncOmniEngine(
|
||||
|
||||
@@ -260,6 +260,37 @@ class TestDiTAttention:
|
||||
assert attention.to_k.out_features == 512
|
||||
assert attention.to_v.out_features == 512
|
||||
|
||||
def test_embedded_dit_honors_diffusion_attention_backend(self, monkeypatch):
|
||||
"""The LLM-hosted code2wav DiT must consume the diffusion env override."""
|
||||
from vllm_omni.diffusion.config import get_current_diffusion_config_or_none
|
||||
from vllm_omni.model_executor.models.cosyvoice3.cosyvoice3_code2wav import _build_dit_estimator
|
||||
|
||||
monkeypatch.setenv("DIFFUSION_ATTENTION_BACKEND", "TORCH_SDPA")
|
||||
|
||||
estimator = _build_dit_estimator(
|
||||
{
|
||||
"dim": 32,
|
||||
"depth": 1,
|
||||
"heads": 2,
|
||||
"dim_head": 16,
|
||||
"dropout": 0.0,
|
||||
"ff_mult": 2,
|
||||
"mel_dim": 8,
|
||||
"mu_dim": 8,
|
||||
"spk_dim": 4,
|
||||
"out_channels": 8,
|
||||
"static_chunk_size": 4,
|
||||
"num_decoding_left_chunks": 1,
|
||||
}
|
||||
)
|
||||
|
||||
attention = estimator.transformer_blocks[0].attn.attn
|
||||
assert attention.backend_pref == "TORCH_SDPA"
|
||||
assert attention.backend_explicit is True
|
||||
assert attention.attn_backend is not None
|
||||
assert attention.attn_backend.get_name() == "SDPA"
|
||||
assert get_current_diffusion_config_or_none() is None
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_forward_passes_2d_mask_without_casting_qkv(self):
|
||||
|
||||
@@ -39,7 +39,10 @@ CHUNK_LEN = 24
|
||||
TOTAL_MEL = 96
|
||||
SPM = 480 # samples per mel frame for the small test HiFT (8*5*3*4), matching the real model
|
||||
|
||||
LONG_TOTAL_MEL = 400 # long enough that the 64-frame window actually truncates
|
||||
WINDOW_LEN = 64
|
||||
# Six chunks exercise three updates after history exceeds the bounded window,
|
||||
# without spending minutes recomputing an unnecessarily long synthetic signal.
|
||||
LONG_TOTAL_MEL = 144
|
||||
PCM16_LSB = 1.0 / 32767.0 # smallest representable PCM16 difference
|
||||
|
||||
# Worst observed deviation is 1.97e-7 (~2x float32 eps); atol=1e-6 keeps 5x headroom.
|
||||
@@ -189,16 +192,19 @@ def test_incremental_hift_bounded_window_is_close(config):
|
||||
def test_incremental_hift_matches_streaming_reference_within_tolerance(config):
|
||||
"""Windowed output matches the full-cumulative reference within float32 rounding.
|
||||
|
||||
The load-bearing correctness test: 400 mel frames far exceeds the 64-frame
|
||||
window, so truncation genuinely exercises the phase/noise carry. Float64
|
||||
agrees to ~1e-16, confirming the residual here is rounding, not truncation.
|
||||
The load-bearing correctness test crosses the 64-frame window repeatedly,
|
||||
so truncation genuinely exercises the phase/noise carry. Float64 agrees to
|
||||
~1e-16, confirming the residual here is rounding, not truncation.
|
||||
"""
|
||||
hift = _make_hift(config)
|
||||
model = _make_model(hift, window_len=64) # the real _hift_window_len
|
||||
model = _make_model(hift, window_len=WINDOW_LEN) # the real _hift_window_len
|
||||
chunks = _chunks(total_mel=LONG_TOTAL_MEL)
|
||||
|
||||
trim = int(hift.f0_predictor.condnet[0].causal_padding)
|
||||
assert LONG_TOTAL_MEL > 64 + trim + CHUNK_LEN # window must actually truncate
|
||||
truncating_steps = sum(
|
||||
history_len > WINDOW_LEN + trim for history_len in range(CHUNK_LEN, LONG_TOTAL_MEL, CHUNK_LEN)
|
||||
)
|
||||
assert truncating_steps >= 3 # exercise repeated truncation, not only the boundary
|
||||
|
||||
full = _full_reference(model, chunks)
|
||||
incr = _incremental(model, chunks)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""VLLM_OMNI_VOXCPM_CODE_PATH still works but warns that it is deprecated (#6232)."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.model_executor.models.voxcpm2 import voxcpm2_import_utils
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _capture_warnings(monkeypatch) -> list[str]:
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(voxcpm2_import_utils.logger, "warning_once", lambda msg, *args: messages.append(msg))
|
||||
return messages
|
||||
|
||||
|
||||
def test_code_path_env_warns_and_is_still_used(monkeypatch, tmp_path: Path):
|
||||
messages = _capture_warnings(monkeypatch)
|
||||
monkeypatch.setenv("VLLM_OMNI_VOXCPM_CODE_PATH", str(tmp_path))
|
||||
|
||||
candidates = voxcpm2_import_utils._iter_voxcpm2_src_candidates()
|
||||
|
||||
assert candidates[0] == tmp_path
|
||||
assert len(messages) == 1
|
||||
assert "VLLM_OMNI_VOXCPM_CODE_PATH is deprecated" in messages[0]
|
||||
|
||||
|
||||
def test_no_warning_without_code_path_env(monkeypatch):
|
||||
messages = _capture_warnings(monkeypatch)
|
||||
monkeypatch.delenv("VLLM_OMNI_VOXCPM_CODE_PATH", raising=False)
|
||||
|
||||
voxcpm2_import_utils._iter_voxcpm2_src_candidates()
|
||||
|
||||
assert messages == []
|
||||
@@ -142,6 +142,37 @@ def test_native_duplex_speak_segment_reaches_split_talker() -> None:
|
||||
assert info["duplex"]["turn_id"] == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize("folded_decisions", [0, 1, 2])
|
||||
@pytest.mark.parametrize("bos_in_prompt", [False, True])
|
||||
def test_native_duplex_tts_bos_aligns_after_window_rebuild(folded_decisions, bos_in_prompt) -> None:
|
||||
prompt_ids = [101] * 10
|
||||
output_ids = [9303] * folded_decisions + [9301, 21, 22, 9308]
|
||||
if bos_in_prompt:
|
||||
prompt_ids[-1] = 9301
|
||||
output_ids = [21, 22, 9308]
|
||||
rows = len(prompt_ids) + len(output_ids) - folded_decisions
|
||||
latent = torch.arange(rows * 4, dtype=torch.float32).reshape(rows, 4)
|
||||
source = _output(
|
||||
prompt_ids=prompt_ids,
|
||||
output_ids=output_ids,
|
||||
latent=latent,
|
||||
multimodal_output={
|
||||
"duplex_prompt_token_ids": prompt_ids,
|
||||
"meta": {
|
||||
"tts_bos_token_id": 9301,
|
||||
"tts_eos_token_id": 9302,
|
||||
"listen_token_id": 9303,
|
||||
"speak_token_id": 9304,
|
||||
"chunk_eos_token_id": 9308,
|
||||
},
|
||||
},
|
||||
)
|
||||
converted = llm2tts([source], prompt=[{}], _streaming_context=SimpleNamespace(bridge_states={}))[0]
|
||||
info = converted["model_intermediate_buffer"]
|
||||
assert info["ids"]["tts"] == [21, 22]
|
||||
torch.testing.assert_close(torch.as_tensor(info["hidden_states"]["tts"]), latent[-3:-1])
|
||||
|
||||
|
||||
def test_native_duplex_continuation_appends_only_new_talker_condition() -> None:
|
||||
prompt_ids = [101, 102]
|
||||
token_ids = {
|
||||
|
||||
@@ -848,6 +848,176 @@ def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format():
|
||||
assert result["prompt_suffix_len"] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [None, "off", "basic", "context"])
|
||||
def test_minicpmo_stage0_history_collection_starts_with_session_config(mode):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import _MiniCPMO45Stage0SessionState
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="history-lifecycle")
|
||||
config = {} if mode is None else {"duplex_window_config": {"sliding_window_mode": mode}}
|
||||
runtime._prepare_session_context(state, {}, runtime_config=config)
|
||||
for seq in range(1, 101):
|
||||
runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=seq)
|
||||
enabled = mode in {"basic", "context"}
|
||||
assert state.window_enabled is enabled
|
||||
assert len(state.window_units) == (99 if enabled else 0)
|
||||
assert (state.pending_window_unit is not None) is enabled
|
||||
assert state.pending_window_generated_tokens == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["basic", "context"])
|
||||
@pytest.mark.parametrize("buffered", [False, True])
|
||||
def test_minicpmo_stage0_multi_chunk_append_matches_scheduler_window(mode, buffered):
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import _MiniCPMO45Stage0SessionState
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="multi-chunk-window", window_enabled=True)
|
||||
if buffered:
|
||||
# Keep the tiny fake processor's initial chunk unpadded to model carry.
|
||||
runtime._pad_first_audio_chunk_if_needed = lambda *args: None
|
||||
assert runtime._stage_prefill_embeddings_only(state, np.zeros(2, dtype=np.float32), seq=0)["success"] is False
|
||||
first = runtime._stage_prefill_embeddings_only(
|
||||
state, np.zeros(6 if buffered else 8, dtype=np.float32), seq=1, final=buffered
|
||||
)
|
||||
assert first["input_token_ids"] == [1, 11, 2, 1, 11]
|
||||
assert state.window_units == []
|
||||
assert state.pending_window_unit.token_ids == first["input_token_ids"]
|
||||
session = SimpleNamespace(num_prompt_tokens=5)
|
||||
replaced = False
|
||||
for seq in (2, 3):
|
||||
info = {
|
||||
"duplex": {
|
||||
"data_plane": True,
|
||||
"seq": seq,
|
||||
"runtime_config": {
|
||||
"duplex_window_config": {
|
||||
"sliding_window_mode": mode,
|
||||
"basic_window_high_tokens": 1,
|
||||
"basic_window_low_tokens": 2,
|
||||
"context_max_units": 1,
|
||||
"context_previous_max_tokens": 16,
|
||||
},
|
||||
"duplex_window_previous_marker_token_ids": [201, 5],
|
||||
},
|
||||
}
|
||||
}
|
||||
update = SimpleNamespace(prompt_token_ids=[3, 2, 1, 11], model_intermediate_buffer=info)
|
||||
planned = OmniARScheduler._prepare_minicpmo45_stage0_window(
|
||||
session, update, segment_output_ids=[44], completed_terminator=3
|
||||
)
|
||||
result = runtime._stage_prefill_embeddings_only(
|
||||
state, np.zeros(4, dtype=np.float32), seq=seq, stage0_window=info["duplex"]["stage0_window"]
|
||||
)
|
||||
assert result["stage0_window_replaced"] is planned
|
||||
if planned:
|
||||
assert result["num_input_tokens"] == len(update.prompt_token_ids)
|
||||
replaced = True
|
||||
break
|
||||
session.num_prompt_tokens += 1 + len(update.prompt_token_ids)
|
||||
assert replaced
|
||||
|
||||
|
||||
def test_minicpmo_stage0_basic_window_rebuilds_from_retained_units():
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-basic-window", window_enabled=True)
|
||||
first = runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
|
||||
assert first["input_token_ids"] == [1, 11]
|
||||
state.pending_window_generated_tokens.append(5)
|
||||
state.pending_terminator_token = 3
|
||||
|
||||
rebuilt = runtime._stage_prefill_embeddings_only(
|
||||
state,
|
||||
np.zeros(4, dtype=np.float32),
|
||||
seq=2,
|
||||
stage0_window={
|
||||
"replace": True,
|
||||
"mode": "basic",
|
||||
"drop_units": 1,
|
||||
"replacement_prompt_len": 2,
|
||||
},
|
||||
)
|
||||
|
||||
assert rebuilt["stage0_window_replaced"] is True
|
||||
assert rebuilt["input_token_ids"] == [1, 11]
|
||||
assert state.window_units == []
|
||||
|
||||
|
||||
def test_minicpmo_stage0_context_window_inserts_previous_before_suffix():
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-context-window", window_enabled=True)
|
||||
prefix = runtime._embed_token(200)
|
||||
suffix = runtime._embed_token(202)
|
||||
state.context_embeds = [prefix, suffix]
|
||||
state.context_token_ids = [200, 202]
|
||||
state.context_prefix_embeds = [prefix]
|
||||
state.context_prefix_token_ids = [200]
|
||||
state.context_suffix_embeds = [suffix]
|
||||
state.context_suffix_token_ids = [202]
|
||||
first = runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
|
||||
assert first["input_token_ids"] == [200, 202, 1, 11]
|
||||
state.pending_window_generated_tokens.append(42)
|
||||
state.pending_terminator_token = 3
|
||||
|
||||
rebuilt = runtime._stage_prefill_embeddings_only(
|
||||
state,
|
||||
np.zeros(4, dtype=np.float32),
|
||||
seq=2,
|
||||
stage0_window={
|
||||
"replace": True,
|
||||
"mode": "context",
|
||||
"drop_units": 1,
|
||||
"previous_token_ids": [42],
|
||||
"previous_marker_token_ids": [201, 5],
|
||||
"replacement_prompt_len": 7,
|
||||
},
|
||||
)
|
||||
|
||||
assert rebuilt["input_token_ids"] == [200, 201, 5, 42, 202, 1, 11]
|
||||
assert rebuilt["num_input_tokens"] == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pending_terminator", [None, 3, 99])
|
||||
def test_minicpmo_stage0_window_uses_accepted_output_not_async_sampler_history(pending_terminator):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
runtime._stage_audio_embeddings = lambda *args, **kwargs: torch.zeros((10, 2))
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-accepted-window", window_enabled=True)
|
||||
state.context_prefix_token_ids = [200] * 7
|
||||
state.context_suffix_token_ids = [202]
|
||||
state.context_token_ids = [*state.context_prefix_token_ids, 202]
|
||||
state.context_prefix_embeds = [runtime._embed_token(token_id) for token_id in state.context_prefix_token_ids]
|
||||
state.context_suffix_embeds = [runtime._embed_token(202)]
|
||||
state.context_embeds = [*state.context_prefix_embeds, *state.context_suffix_embeds]
|
||||
runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
|
||||
for seq in (2, 3):
|
||||
state.pending_window_generated_tokens = [3, 40, 41]
|
||||
state.pending_terminator_token = pending_terminator
|
||||
plan = {"completed_token_ids": [], "completed_terminator_token_id": 3}
|
||||
if seq == 3:
|
||||
plan.update(replace=True, mode="context", drop_units=1, replacement_prompt_len=32)
|
||||
result = runtime._stage_prefill_embeddings_only(
|
||||
state, np.zeros(4, dtype=np.float32), seq=seq, stage0_window=plan
|
||||
)
|
||||
|
||||
assert result["num_input_tokens"] == 32
|
||||
assert [len(unit.token_ids) for unit in state.window_units] == [13]
|
||||
assert len(state.pending_window_unit.token_ids) == 11
|
||||
assert 40 not in result["input_token_ids"]
|
||||
assert 41 not in result["input_token_ids"]
|
||||
|
||||
|
||||
def _stage0_vision_runtime():
|
||||
import torch
|
||||
|
||||
|
||||
Reference in new issue
Block a user