Merge branch 'main' into qwen-image-2.1

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py authored and GitHub committed 2026-09-22 10:11:12 +08:00
commit 8e0695f362
51 files changed
+2189 -70

No files matched your search

+22 -2
View File
@@ -67,11 +67,31 @@ def test_ready_diffusion_cpu_suite_is_sharded() -> None:
assert "--shard-id=$$BUILDKITE_PARALLEL_JOB" in pytest_command
def test_cosyvoice_gpu_abort_gets_one_fresh_job_retry() -> None:
step = _find_step("CosyVoice3-TTS E2E Test", AMD_READY_PIPELINE)
def test_cosyvoice_ready_smoke_uses_sdpa() -> None:
step = _find_step("CosyVoice3-TTS E2E Smoke (SDPA)", AMD_READY_PIPELINE)
assert step["grade"] == "Blocking"
assert step["retry"] == {"automatic": [{"exit_status": 134, "limit": 1}]}
assert "export DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA" in step["commands"]
pytest_command = next(command for command in step["commands"] if "pytest" in command)
assert "tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py::test_voice_clone_zh_002" in pytest_command
def test_cosyvoice_full_default_backend_suite_runs_nightly() -> None:
step = _find_step("CosyVoice3-TTS E2E Test", AMD_NIGHTLY_PIPELINE)
assert step["grade"] == "NonBlocking"
assert step["timeout_in_minutes"] == 90
assert step["retry"] == {"automatic": [{"exit_status": 134, "limit": 1}]}
assert all("DIFFUSION_ATTENTION_BACKEND" not in command for command in step["commands"])
pytest_command = next(command for command in step["commands"] if "pytest" in command)
assert "tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py" in pytest_command
assert "::" not in pytest_command
def test_amd_template_preserves_step_retry_policy() -> None:
template = AMD_TEMPLATE.read_text(encoding="utf-8")
# Both grouped and top-level AMD steps must preserve an explicit retry
# policy when the source suite is rendered into the uploaded pipeline.
@@ -69,6 +69,39 @@ def _make_update(prompt_token_ids: list[int] | None = None) -> StreamingUpdate:
)
def _make_minicpm_window_update(
*,
seq: int,
mode: str,
high: int = 15,
low: int = 10,
context_max_units: int = 1,
) -> StreamingUpdate:
update = _make_update([0] * 8)
update.model_intermediate_buffer = {
"duplex": {
"data_plane": True,
"seq": seq,
"runtime_config": {
"duplex_scheduler_token_id": 0,
"duplex_first_append_context_tokens": 3,
"duplex_window_prefix_tokens": 2,
"duplex_window_suffix_token_ids": [3],
"duplex_window_previous_marker_token_ids": [70, 71],
"duplex_window_special_token_ids": [99],
"duplex_window_config": {
"sliding_window_mode": mode,
"basic_window_high_tokens": high,
"basic_window_low_tokens": low,
"context_previous_max_tokens": 4,
"context_max_units": context_max_units,
},
},
}
}
return update
def _make_talker_adapter(
*,
max_model_len: int = 100,
@@ -510,6 +543,225 @@ def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder()
assert sched._new_prompt_len_snapshot[session.request_id] == 2
def test_stage0_basic_window_rebuilds_below_low_watermark() -> None:
sched = _make_scheduler(stage_id=0)
session = _make_request()
session.prompt_token_ids = [0] * 9
session._all_token_ids.clear()
session._all_token_ids.extend(session.prompt_token_ids)
session.num_prompt_tokens = 9
session.append_output_token_ids([40])
session.num_computed_tokens = 10
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
update = _make_minicpm_window_update(seq=2, mode="basic")
sched._update_request_as_session(session, update)
assert session.prompt_token_ids == [0] * 9
assert session.num_computed_tokens == 0
assert update.model_intermediate_buffer["meta"]["replace_streaming_prompt"] is True
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
assert plan == {
"completed_token_ids": [40],
"replace": True,
"mode": "basic",
"drop_units": 1,
"dropped_tokens": 9,
"previous_token_ids": [],
"previous_marker_token_ids": [70, 71],
"replacement_prompt_len": 9,
}
sched._free_request_blocks.assert_called_once_with(session)
def test_stage0_context_window_compacts_dropped_speech() -> None:
sched = _make_scheduler(stage_id=0)
session = _make_request()
session.prompt_token_ids = [0] * 9
session._all_token_ids.clear()
session._all_token_ids.extend(session.prompt_token_ids)
session.num_prompt_tokens = 9
session.append_output_token_ids([40])
session.num_computed_tokens = 10
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
first = _make_minicpm_window_update(seq=2, mode="context")
sched._update_request_as_session(session, first)
assert session.num_prompt_tokens == 18
assert first.model_intermediate_buffer["duplex"]["stage0_window"] == {"completed_token_ids": [40]}
session.append_output_token_ids([50])
session.num_computed_tokens = 19
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
second = _make_minicpm_window_update(seq=3, mode="context")
sched._update_request_as_session(session, second)
assert session.prompt_token_ids == [0] * 21
plan = second.model_intermediate_buffer["duplex"]["stage0_window"]
assert plan["completed_token_ids"] == [50]
assert plan["drop_units"] == 1
assert plan["previous_token_ids"] == [40]
# The worker embeds this exact marker, not a re-tokenized one, so the
# previous region is len(marker) + len(previous).
assert plan["previous_marker_token_ids"] == [70, 71]
assert plan["replacement_prompt_len"] == 21
assert getattr(session, "_minicpmo45_window_previous_len") == 3
@pytest.mark.parametrize("cleared_outputs", [False, True])
@pytest.mark.parametrize("in_flight", [0, 1])
def test_stage0_window_uses_confirmed_span_and_terminator(cleared_outputs, in_flight) -> None:
sched = _make_scheduler(stage_id=0)
session = _make_request()
session.prompt_token_ids = [0] * 9
session._all_token_ids[:] = session.prompt_token_ids
session.num_prompt_tokens = 9
session.append_output_token_ids([40, 99])
session.num_computed_tokens = 10 + in_flight
session.num_output_placeholders = in_flight
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
if cleared_outputs:
# A stop without a queued append clears this before the next update.
session._output_token_ids.clear()
update = _make_minicpm_window_update(seq=2, mode="basic")
sched._update_request_as_session(session, update)
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
assert plan["completed_token_ids"] == [40]
assert plan["completed_terminator_token_id"] == 99
assert plan["replacement_prompt_len"] == 9
assert plan["dropped_tokens"] == 9
def test_stage0_window_rebuild_that_overflows_max_model_len_finishes_the_session() -> None:
"""A rebuilt window prompt is bounded by the client's window settings, not
by the model: a unit carrying camera frames is hundreds of tokens, and a
``context`` window adds the ``previous`` region on top. The replacement
branch must therefore check ``replacement_prompt_len`` against
``max_model_len - sample_room`` and finish the session with
``context_length_exceeded``, exactly like a plain extension."""
sched = _make_live_session_scheduler(max_model_len=21)
session = _make_request()
session.prompt_token_ids = [0] * 9
session._all_token_ids[:] = [0] * 9
session.num_prompt_tokens = 9
session._output_token_ids[:] = [40]
session._all_token_ids.append(40)
session.num_computed_tokens = 10
session.num_output_placeholders = 0
_park_session(sched, session)
# The parked append state above is the frame before this update: one
# confirmed output token, fully computed.
session._output_token_ids[:] = [40]
session._all_token_ids[:] = [*session.prompt_token_ids, 40]
session.num_computed_tokens = 10
session.num_in_flight_tokens = 0
sched.num_sampled_tokens_per_step = 1
# The replacement plan rebuilds 21 tokens (retained context plus the
# append) and one step samples one more, so 21 leaves no room.
update = _make_minicpm_window_update(seq=2, mode="basic")
update.prompt_token_ids = [0] * 20
original_prompt = list(session.prompt_token_ids)
sched._update_request_as_session(session, update)
assert update.model_intermediate_buffer["duplex"]["stage0_window"]["replacement_prompt_len"] == 21
assert session.prompt_token_ids == original_prompt
assert session.num_prompt_tokens == 9
assert session.status == RequestStatus.FINISHED_ERROR
assert session.request_id not in sched.requests
assert sched.finished_req_ids == {session.request_id}
sched._free_request_blocks.assert_called_once_with(session)
client_index, reason = sched._streaming_context_overflow[session.request_id]
assert client_index == session.client_index
assert reason.startswith("context_length_exceeded: ")
assert "21 tokens" in reason and "max_model_len 21" in reason
engine_core_outputs = _run_idle_step(sched)
(output,) = engine_core_outputs[session.client_index].outputs
assert output.request_id == session.request_id
assert output.finish_reason == FinishReason.ERROR
assert output.stop_reason == reason
def test_stage0_window_rebuild_that_leaves_room_to_sample_replaces_the_prompt() -> None:
"""One slot above the plan fits: the replacement applies through the normal
replacement path."""
sched = _make_scheduler(stage_id=0)
sched.max_model_len = 25
sched.num_sampled_tokens_per_step = 1
session = _make_request()
session.prompt_token_ids = [0] * 9
session._all_token_ids[:] = session.prompt_token_ids
session.num_prompt_tokens = 9
session.append_output_token_ids([40])
session.num_computed_tokens = 10
session.num_output_placeholders = 0
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
update = _make_minicpm_window_update(seq=2, mode="basic")
update.prompt_token_ids = [0] * 20
sched._update_request_as_session(session, update)
assert update.model_intermediate_buffer["meta"]["replace_streaming_prompt"] is True
assert update.model_intermediate_buffer["duplex"]["stage0_window"]["replacement_prompt_len"] == 21
assert session.prompt_token_ids == [0] * 21
assert session.num_prompt_tokens == 21
assert session.num_computed_tokens == 0
assert session.status == RequestStatus.WAITING
sched._free_request_blocks.assert_called_once_with(session)
assert not getattr(sched, "_streaming_context_overflow", {})
def test_stage0_window_open_start_zero_is_not_replaced_by_the_context_reserve() -> None:
"""A recorded ``open_start`` of 0 is a legitimate empty context prefix. The
fallback must be an explicit ``is None`` check: treating 0 as missing
substitutes the context reserve, counts the suffix twice, and the worker's
rebuild-length check raises on the first replacement."""
session = SimpleNamespace(
num_prompt_tokens=9,
_all_token_ids=[0] * 9 + [40],
num_computed_tokens=10,
num_output_placeholders=0,
_minicpmo45_window_open_start=0,
)
update = _make_minicpm_window_update(seq=2, mode="basic")
update.prompt_token_ids = [0] * 21
assert OmniARScheduler._prepare_minicpmo45_stage0_window(
session, update, segment_output_ids=[40], completed_terminator=3
)
# open_start stays 0, so the reported unit spans the whole 12-token
# boundary instead of the 9 the context reserve would fold in.
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
assert plan["dropped_tokens"] == 12
assert plan["replacement_prompt_len"] == 19
def test_stage0_window_open_start_falls_back_to_the_context_reserve_when_unset() -> None:
session = SimpleNamespace(
num_prompt_tokens=9,
_all_token_ids=[0] * 9 + [40],
num_computed_tokens=10,
num_output_placeholders=0,
)
update = _make_minicpm_window_update(seq=2, mode="basic")
update.prompt_token_ids = [0] * 21
assert OmniARScheduler._prepare_minicpmo45_stage0_window(
session, update, segment_output_ids=[40], completed_terminator=3
)
# fallback open_start is preserve_len 3 from duplex_first_append_context_tokens,
# so the unit is three tokens shorter.
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
assert plan["dropped_tokens"] == 9
assert plan["replacement_prompt_len"] == 22
def test_explicit_streaming_payload_replaces_placeholder_prompt() -> None:
sched = _make_scheduler(stage_id=1)
sched.chunk_transfer_adapter = SimpleNamespace(
+108
View File
@@ -60,6 +60,7 @@ def test_custom_cache_dit_enablers_are_registered_explicitly():
"Cosmos3OmniPipeline": cd_model_specific.enable_cache_for_cosmos3,
"Krea2Pipeline": cd_model_specific.enable_cache_for_krea2,
"Magi2Pipeline": cd_model_specific.enable_cache_for_magi2,
"MammothModa2DiTPipeline": cd_model_specific.enable_cache_for_mammothmoda2,
}
with patch.dict(cd_backend.CUSTOM_DIT_ENABLERS, {}, clear=True):
@@ -116,6 +117,33 @@ def test_magi2_cache_dit_targets_only_nested_repeated_layers(mock_block_adapter,
assert get_transformer(pipeline) is transformer_block
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.enable_cache_for_dit")
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter")
def test_mammothmoda2_cache_dit_targets_only_main_layers(mock_block_adapter, mock_enable_cache):
pipeline = Mock()
transformer = pipeline.gen_transformer
layers = torch.nn.ModuleList([torch.nn.Identity()])
transformer.layers = layers
adapter = mock_block_adapter.return_value
refresh = Mock()
mock_enable_cache.return_value = refresh
result = cd_model_specific.enable_cache_for_mammothmoda2(pipeline, SAMPLE_CACHE_CONFIG)
mock_block_adapter.assert_called_once()
adapter_kwargs = mock_block_adapter.call_args.kwargs
assert adapter_kwargs["transformer"] is transformer
assert adapter_kwargs["blocks"] == [layers]
assert adapter_kwargs["has_separate_cfg"] is True
assert adapter_kwargs["check_forward_pattern"] is True
assert result.refresh is refresh
assert result.targets == (adapter,)
get_transformer = mock_enable_cache.call_args.kwargs["get_pipeline_transformer"]
assert get_transformer(pipeline) is transformer
# Sequential-CFG parity is mandatory while cache-dit is installed.
assert pipeline._cache_dit_requires_paired_cfg is True
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit.summary")
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter.is_cached", return_value=True)
def test_cache_summary_uses_custom_nested_targets(mock_is_cached, mock_summary):
@@ -338,3 +366,83 @@ def test_summary_with_no_transformer_is_nonfatal():
pass
cache_summary(pipeline=FakePipeline())
# This test is skipped on ROCm since rocm_unquantized_gemm doesn't support CPU backend
@pytest.mark.skipif(
current_omni_platform.is_rocm(),
reason="vLLM ROCm custom ops lack CPU fallback",
)
def test_mammothmoda2_cache_dit_runs_end_to_end_on_tiny_model(request: pytest.FixtureRequest):
"""A tiny MammothModa2 DiT runs through Cache-DiT with paired CFG forwards.
Exercises the Pattern_3 contract end to end: after enabling the custom
enabler, blocks must be callable as ``block(hidden_states, **kwargs)``
and the pipeline's two-forwards-per-step CFG cadence must drive the
cache context without shape or parity errors.
"""
from cache_dit import BlockAdapter
from vllm_omni.diffusion.models.mammoth_moda2.mammothmoda2_dit_model import Transformer2DModel
from vllm_omni.diffusion.models.mammoth_moda2.rope_real import RotaryPosEmbedReal
# MammothModa2 attention resolves through the shared Omni attention layer,
# which picks the platform-default backend at construction time. Pin
# TORCH_SDPA so a CUDA-visible host does not select FA3 for CPU tensors.
with _force_torch_sdpa():
model = Transformer2DModel(
patch_size=2,
in_channels=4,
hidden_size=96,
num_layers=4,
num_refiner_layers=1,
num_attention_heads=2,
num_kv_heads=2,
multiple_of=8,
axes_dim_rope=(16, 16, 16),
axes_lens=(300, 512, 512),
text_feat_dim=16,
)
model.eval()
MammothModa2Pipeline = type("MammothModa2DiTPipeline", (), {})
pipeline = MammothModa2Pipeline()
pipeline.gen_transformer = model
backend = CacheDiTBackend(DiffusionCacheConfig())
backend.enable(pipeline)
request.addfinalizer(lambda: backend.disable(pipeline))
assert BlockAdapter.is_cached(model)
num_inference_steps = 8
backend.refresh(pipeline, num_inference_steps=num_inference_steps)
freqs_cis = RotaryPosEmbedReal.get_freqs_real((16, 16, 16), (300, 512, 512), theta=10000)
latents = torch.randn(1, 4, 8, 8)
text = torch.randn(1, 5, 16)
negative = torch.randn(1, 3, 16)
text_mask = torch.ones(1, 5, dtype=torch.bool)
negative_mask = torch.ones(1, 3, dtype=torch.bool)
with torch.no_grad():
for step in range(num_inference_steps):
timestep = torch.full((1,), 1.0 - step / num_inference_steps)
# Conditional pass ...
cond = model(
hidden_states=latents,
timestep=timestep,
text_hidden_states=text,
freqs_cis=freqs_cis,
text_attention_mask=text_mask,
)
# ... always followed by the unconditional pass (parity).
uncond = model(
hidden_states=latents,
timestep=timestep,
text_hidden_states=negative,
freqs_cis=freqs_cis,
text_attention_mask=negative_mask,
)
latents = uncond + 4.0 * (cond - uncond)
assert latents.shape == (1, 4, 8, 8)
assert torch.isfinite(latents).all()
@@ -1,13 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from contextlib import contextmanager
from dataclasses import dataclass
from types import SimpleNamespace
import cache_dit
import pytest
import torch
from cache_dit import BlockAdapter
from torch import nn
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig, TransformerConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTBackend, RequestScopedCacheDiTRuntime
from vllm_omni.diffusion.cache.selector import get_cache_backend
from vllm_omni.diffusion.config import set_current_diffusion_config
from vllm_omni.diffusion.data import (
AttentionConfig,
DiffusionCacheConfig,
DiffusionOutput,
OmniDiffusionConfig,
TransformerConfig,
)
from vllm_omni.diffusion.models.interface import adopt_request_scoped_cache_dit
from vllm_omni.diffusion.models.mammoth_moda2 import pipeline_mammothmoda2_dit
from vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit import (
MammothModa2DiTPipeline,
@@ -17,6 +31,7 @@ from vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit import (
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -110,6 +125,10 @@ def _pipeline_shell() -> MammothModa2DiTPipeline:
pipeline.device = torch.device("cpu")
pipeline.config = _build_mammoth_config(_od_config())
pipeline._llm_hidden_size = 8
# Cache-DiT runtime state that __init__ normally creates (disabled here,
# matching a pipeline built without a cache_backend).
pipeline._cache_dit_runtime = RequestScopedCacheDiTRuntime(pipeline)
pipeline._cache_dit_config = None
return pipeline
@@ -118,12 +137,14 @@ def _batch(
request_id: str = "req-7",
prompt: object | None = None,
sampling: OmniDiffusionSamplingParams | None = None,
height: int = 32,
width: int = 48,
) -> DiffusionRequestBatch:
if prompt is None:
prompt = {
"prompt": "",
"height": 32,
"width": 48,
"height": height,
"width": width,
"additional_information": {
"full_hidden_states": torch.arange(32, dtype=torch.float32).reshape(4, 8),
"full_token_ids": [10, 11, 100, 101],
@@ -390,3 +411,162 @@ def test_forward_rejects_missing_visual_tokens_before_model_access() -> None:
}
with pytest.raises(ValueError, match="no visual-token hidden states.*req-empty"):
_pipeline_shell().forward(_batch(request_id="req-empty", prompt=prompt))
@contextmanager
def _force_torch_sdpa():
"""Pin TORCH_SDPA so CPU shape tests do not pick CUDA-only backends (FA3)."""
od_config = SimpleNamespace(
diffusion_attention_config=AttentionConfig(default="TORCH_SDPA"),
parallel_config=SimpleNamespace(ring_degree=1),
)
with set_current_diffusion_config(od_config):
yield
def _constructible_raw_config() -> dict:
"""Checkpoint config whose DiT/VAE are small enough to actually run on CPU."""
return {
"model_type": "mammothmoda2",
"llm_config": {
"model_type": "mammothmoda2_qwen2_5_vl",
"text_config": {
"model_type": "mammothmoda2_qwen2_5_vl_text",
"hidden_size": 8,
"gen_vocab_start_index": 100,
},
},
"gen_vae_config": {"block_out_channels": [8, 8], "norm_num_groups": 8},
"gen_dit_config": {
"patch_size": 2,
"in_channels": 4,
"hidden_size": 96,
"num_layers": 4,
"num_refiner_layers": 1,
"num_attention_heads": 2,
"num_kv_heads": 2,
"multiple_of": 8,
"axes_dim_rope": [16, 16, 16],
"axes_lens": [300, 512, 512],
"text_feat_dim": 16,
},
# Top-level axes feed the pipeline-owned rotary table (see pipeline __init__).
"gen_axes_dim_rope": [16, 16, 16],
"gen_axes_lens": [300, 512, 512],
}
def _cache_dit_od_config() -> OmniDiffusionConfig:
return OmniDiffusionConfig(
model="/models/MammothModa2-Preview",
model_class_name="MammothModa2DiTPipeline",
tf_model_config=TransformerConfig.from_dict(_constructible_raw_config()),
cache_backend="cache_dit",
cache_config=DiffusionCacheConfig(),
)
@pytest.mark.skipif(
current_omni_platform.is_rocm(),
reason="vLLM ROCm custom ops lack CPU fallback",
)
def test_forward_transitions_request_scoped_cache_dit_across_requests(request: pytest.FixtureRequest) -> None:
"""Cache-DiT request lifecycle on one production-built pipeline instance.
Mirrors the diffusion runner startup (``get_cache_backend`` → ``enable`` →
``adopt_request_scoped_cache_dit``) on a tiny pipeline built through the
real ``__init__``, then drives three requests through the real ``forward()``:
1. a partial-CFG request installs the request-scoped cache context:
cond/uncond forwards stay paired on every denoise step (parity
accounting), even outside ``cfg_range``, and blocks record cached steps;
2. a guidance=1.0 request runs with hooks disabled (``prepare(None)``) and
its output is bit-identical to a reference captured before the backend
was ever enabled, proving a clean uninstall with no stale module state;
3. a CFG request with a different step count and image size re-installs the
hooks through a fresh context, so no stale shape or step count survives.
"""
torch.manual_seed(0) # Deterministic random init → deterministic cache hits.
with _force_torch_sdpa():
pipeline = MammothModa2DiTPipeline(od_config=_cache_dit_od_config())
pipeline.eval()
assert pipeline._cache_dit_config is not None
request.addfinalizer(pipeline._cache_dit_runtime.disable)
# The default prompt carries 2 text + 2 visual condition tokens, so the
# conditional pass sees 4 tokens and the uncond pass (empty prompt) sees 0.
cond_tokens, uncond_tokens = 4, 0
seq_lens: list[int] = []
def record_call(_module, _args, kwargs):
text = kwargs.get("text_hidden_states")
seq_lens.append(int(text.shape[1]) if text is not None else -1)
hook = pipeline.gen_transformer.register_forward_pre_hook(record_call, with_kwargs=True)
unguided_sampling = OmniDiffusionSamplingParams(
height=32, width=48, seed=4321, guidance_scale=1.0, num_inference_steps=8
)
partial_cfg_sampling = OmniDiffusionSamplingParams(
height=32,
width=48,
seed=1234,
guidance_scale=4.0,
num_inference_steps=8,
extra_args={"cfg_range": [0.2, 0.8]},
)
changed_cfg_sampling = OmniDiffusionSamplingParams(
height=48,
width=64,
seed=777,
guidance_scale=6.0,
num_inference_steps=10,
extra_args={"cfg_range": [0.1, 0.9]},
)
# Reference run for phase 2: identical request while hooks were never
# installed on this pipeline.
assert not pipeline.is_cache_dit_enabled()
reference = pipeline.forward(_batch(sampling=unguided_sampling)).output
assert seq_lens == [cond_tokens] * 8
assert torch.isfinite(reference.float()).all()
# Runner startup: enable once, then the pipeline owns all transitions.
backend = get_cache_backend("cache_dit", DiffusionCacheConfig())
assert isinstance(backend, CacheDiTBackend)
backend.enable(pipeline)
assert adopt_request_scoped_cache_dit(pipeline, backend)
assert pipeline.is_cache_dit_enabled()
# Phase 1: partial CFG keeps cond/uncond paired on every step, including
# the steps whose normalized index falls outside cfg_range [0.2, 0.8].
seq_lens.clear()
pipeline.forward(_batch(sampling=partial_cfg_sampling))
assert seq_lens == [cond_tokens, uncond_tokens] * 8
assert pipeline.is_cache_dit_enabled()
assert BlockAdapter.is_cached(pipeline.gen_transformer)
hits = cache_dit.summary(pipeline.gen_transformer, logging=False)[0]
assert hits.cached_steps or hits.cfg_cached_steps
# Phase 2: no-CFG request disables the hooks and must reproduce the
# never-enabled reference bit-for-bit.
seq_lens.clear()
unguided = pipeline.forward(_batch(sampling=unguided_sampling)).output
assert seq_lens == [cond_tokens] * 8
assert not pipeline.is_cache_dit_enabled()
assert not BlockAdapter.is_cached(pipeline.gen_transformer)
assert torch.equal(unguided, reference)
# Phase 3: CFG with a new step count and image size re-installs the hooks
# on a fresh context; output shape follows the new request geometry.
seq_lens.clear()
regenerated = pipeline.forward(_batch(sampling=changed_cfg_sampling, height=48, width=64)).output
assert seq_lens == [cond_tokens, uncond_tokens] * 10
assert pipeline.is_cache_dit_enabled()
assert BlockAdapter.is_cached(pipeline.gen_transformer)
hits = cache_dit.summary(pipeline.gen_transformer, logging=False)[0]
assert hits.cached_steps or hits.cfg_cached_steps
assert regenerated.shape == (1, 3, 12, 16)
assert torch.isfinite(regenerated.float()).all()
hook.remove()
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Drive complete native duplex turns while Stage-0 repeatedly rebuilds KV."""
from __future__ import annotations
import asyncio
import base64
from pathlib import Path
from vllm_omni.clients.duplex import DuplexClient, EventCollector, read_pcm16_wav, wait_for_condition
from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
async def run_window_turn(
*,
url: str,
model: str,
input_wav: Path,
mode: str,
ref_audio: Path | None = None,
repeats: int = 2,
video_frames: list[str] | None = None,
buffered_flush: bool = False,
timeout_s: float = 240.0,
) -> dict[str, object]:
"""Use aggressive limits to exercise multiple windows in one audio turn."""
pcm = read_pcm16_wav(input_wav) * repeats
collector = EventCollector()
reference = (
"data:audio/wav;base64," + base64.b64encode(ref_audio.read_bytes()).decode("ascii")
if ref_audio is not None
else None
)
config = create_duplex_session_config(
modalities=("text", "audio") if ref_audio is not None else ("text",),
ref_audio=reference,
temperature=0.0,
extra_body={
"sliding_window_mode": mode,
"context_max_units": 1,
"context_previous_max_tokens": 16,
"basic_window_high_tokens": 64,
"basic_window_low_tokens": 32,
},
)
client = DuplexClient(
url, model=model, config=config, reconnect=None, heartbeat_interval_s=None, handshake_timeout_s=timeout_s
)
async with client:
reader = asyncio.create_task(collector.consume(client))
collector.add({"type": "session.created", "session": client.session_info})
try:
frames_sent = await client.stream_pcm(
pcm,
chunk_ms=2500 if buffered_flush else 200,
realtime=not buffered_flush,
is_speech=True,
video_frames=video_frames,
)
await client.commit(final=True)
await wait_for_condition(
lambda: collector.count("input_audio_buffer.committed") > 0 or bool(collector.errors()),
timeout_s=timeout_s,
label="final input commit",
)
await wait_for_condition(
lambda: collector.count("response.done") > 0 or bool(collector.errors()),
timeout_s=timeout_s,
label="completed windowed response",
)
# Final execution follows the commit acknowledgement asynchronously.
await asyncio.sleep(3.0)
await client.close(timeout_s=timeout_s)
await asyncio.wait_for(reader, timeout=5.0)
finally:
if not reader.done():
reader.cancel()
await asyncio.gather(reader, return_exceptions=True)
return {
"created": collector.count("session.created"),
"committed": collector.count("input_audio_buffer.committed"),
"done": collector.count("response.done"),
"closed": collector.count("session.closed"),
"errors": collector.errors(),
"audio_bytes": len(collector.audio_bytes()),
"frames_sent": frames_sent,
"input_seconds": len(pcm) / 32000,
}
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Real-model regression for the scheduler-owned Thinker sliding window."""
import asyncio
import pytest
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
SERVER_PARAMS,
duplex_camera_frames,
realtime_url,
resolve_ref_audio,
validated_input_wav,
)
from tests.e2e.online_serving.helpers.minicpmo_window_e2e import run_window_turn
from tests.helpers.mark import hardware_test
pytestmark = [pytest.mark.omni, pytest.mark.advanced_model]
def _assert_complete(result, *, require_audio=False):
assert result["created"] == 1
assert result["committed"] > 0
assert result["done"] > 0
assert result["closed"] == 1
assert result["errors"] == []
if require_audio:
assert result["audio_bytes"] > 0
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
@pytest.mark.parametrize("mode,reference", [("basic", False), ("context", False), ("context", True)])
def test_window_rebuild_and_next_session(omni_server, mode, reference):
# Two independent sessions exercise admission after final-unit execution.
for _ in range(2):
result = asyncio.run(
run_window_turn(
url=realtime_url(omni_server),
model=omni_server.model,
input_wav=validated_input_wav(),
mode=mode,
ref_audio=resolve_ref_audio() if reference else None,
)
)
_assert_complete(result, require_audio=reference)
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
@pytest.mark.parametrize("mode,camera", [("basic", False), ("context", False), ("context", True)])
def test_window_continuous_input(omni_server, tmp_path, mode, camera):
# 24 repeats cover over two minutes and many canonical KV replacements.
frames = duplex_camera_frames(seconds=4, cache_dir=tmp_path / "camera") if camera else None
result = asyncio.run(
run_window_turn(
url=realtime_url(omni_server),
model=omni_server.model,
input_wav=validated_input_wav(),
mode=mode,
repeats=24,
ref_audio=resolve_ref_audio(),
video_frames=frames,
)
)
_assert_complete(result, require_audio=True)
assert isinstance(result["input_seconds"], (int, float)) and result["input_seconds"] > 120
if camera:
assert isinstance(result["frames_sent"], int) and result["frames_sent"] >= 120
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
@pytest.mark.parametrize("mode", ["off", "basic", "context"])
def test_window_buffered_flush(omni_server, mode):
# Burst appends cross processor chunk boundaries and leave a final tail.
result = asyncio.run(
run_window_turn(
url=realtime_url(omni_server),
model=omni_server.model,
input_wav=validated_input_wav(),
mode=mode,
ref_audio=resolve_ref_audio(),
buffered_flush=True,
)
)
_assert_complete(result, require_audio=True)
@@ -0,0 +1,117 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import base64
from types import SimpleNamespace
import pytest
from vllm_omni.engine.duplex.config import DuplexSessionConfig
from vllm_omni.engine.duplex.contracts import DuplexFence
from vllm_omni.model_executor.models.minicpmo_4_5.duplex import plugin as module
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _plugin():
return module.MiniCPMO45DuplexPlugin(lambda *args: None)
@pytest.mark.asyncio
async def test_window_configuration_is_prepared_by_plugin(monkeypatch):
tokenizer = SimpleNamespace(
all_special_ids=[90],
unk_token_id=-1,
eos_token_id=99,
encode=lambda text, add_special_tokens=False: [ord(char) for char in text],
convert_tokens_to_ids=lambda token: {"<|listen|>": 91}.get(token, -1),
)
monkeypatch.setattr(module, "_load_tokenizer", lambda config: tokenizer)
config = DuplexSessionConfig(
modalities=("text",),
extra_body={
"sliding_window_mode": "basic",
"basic_window_high_tokens": 120,
"basic_window_low_tokens": 80,
},
)
runtime = await _plugin().prepare_runtime_config(config, model_config=None)
assert runtime["duplex_window_config"]["sliding_window_mode"] == "basic"
assert "sliding_window_mode" not in config.extra_body
assert runtime["duplex_window_prefix_tokens"] > 0
assert runtime["duplex_window_suffix_token_ids"]
assert runtime["duplex_window_previous_marker_token_ids"]
@pytest.mark.asyncio
async def test_invalid_window_configuration_is_rejected():
config = DuplexSessionConfig(
modalities=("text",),
extra_body={
"basic_window_high_tokens": 80,
"basic_window_low_tokens": 80,
},
)
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError) as exc:
await _plugin().prepare_runtime_config(config, model_config=None)
assert exc.value.code == "invalid_sliding_window_config"
def test_window_update_cannot_change_created_configuration():
config = DuplexSessionConfig(extra_body={"sliding_window_mode": "basic"})
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError) as exc:
_plugin().runtime_config_for_update(config, {"duplex_window_config": {"sliding_window_mode": "off"}})
assert exc.value.code == "sliding_window_update_unsupported"
def test_window_internal_configuration_is_server_owned():
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError):
_plugin().validate_client_extra_body({"duplex_window_config": {}})
@pytest.mark.parametrize("final", [False, True])
@pytest.mark.parametrize("seq,samples,expected", [(1, 32000, 16), (1, 48000, 28), (2, 16000, 13)])
def test_first_and_final_append_reserve_exact_window_input(seq, samples, expected, final):
prompt = module.build_duplex_data_plane_prompt(
request_id="window-request",
fence=DuplexFence("sid", turn_id=1),
session_config={},
runtime_config={"duplex_first_append_context_tokens": 5},
seq=seq,
turn_seq=seq,
payload={
"audio": base64.b64encode(bytes(samples * 4)).decode(),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
},
final=final,
)
assert len(prompt["prompt_token_ids"]) == expected
@pytest.mark.parametrize("seq", [1, 2, 3])
def test_final_exact_chunk_append_reserves_one_unit_not_two(seq):
"""A final append whose audio is already an exact number of chunks must
reserve exactly what Stage0 feeds: one unit per chunk (``<unit>`` plus its
audio) and the closure pair for every unit after the first. Serving pads
the final residual itself, so the old extra 12-slot silent-unit
reservation only put 12 pad positions ahead of the audio in the KV."""
def _budget(*, seq, final):
prompt = module.build_duplex_data_plane_prompt(
request_id="final-request",
fence=DuplexFence("sid", turn_id=1),
session_config={},
runtime_config={"duplex_first_append_context_tokens": 0},
seq=seq,
turn_seq=seq,
payload={
"audio": base64.b64encode(bytes(16000 * 4)).decode(),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
},
final=final,
)
return len(prompt["prompt_token_ids"])
assert _budget(seq=seq, final=True) == _budget(seq=seq, final=False)
@@ -51,6 +51,7 @@ from vllm_omni.engine.duplex.plugin import (
PcmAppendReservation,
)
from vllm_omni.engine.duplex.session import manager as session_manager_module
from vllm_omni.engine.duplex.session.context import DuplexSessionTasks
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession
from vllm_omni.engine.duplex.session.lease import DuplexLeaseActivity
from vllm_omni.engine.duplex.session.manager import DuplexSessionManager
@@ -1612,3 +1613,34 @@ async def test_reaper_loop_survives_one_cleanup_failure(first_cleanup_delay: flo
finally:
shutdown.set()
await asyncio.wait_for(task, timeout=5.0)
# --------------------------------------------------------------------------- #
# Cancellation waits #
# --------------------------------------------------------------------------- #
async def test_cancel_append_tasks_absorbs_a_task_that_outlives_the_wait() -> None:
"""The wait after cancelling is allowed to time out; that is not an error.
On Python 3.10 the timeout arrives as ``asyncio.TimeoutError``, which is a
different class from the builtin ``TimeoutError`` until 3.11.
"""
async def outlives_the_first_cancel() -> bool:
try:
await asyncio.sleep(5)
except asyncio.CancelledError:
await asyncio.sleep(1)
raise
return True
tasks = DuplexSessionTasks()
task = asyncio.create_task(outlives_the_first_cancel())
await asyncio.sleep(0)
tasks.track_append_task(task, epoch=0, final=False, response_bound=False)
tasks.append_tail = task
assert await tasks.cancel_append_tasks(timeout_s=0.05) is True
assert tasks.append_tail is None
await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), timeout=5.0)
@@ -771,6 +771,41 @@ async def test_cancel_response_aborts_the_stage_request_and_reports_playback() -
await close_harness(h)
@pytest.mark.asyncio
async def test_cancel_response_absorbs_a_response_task_that_outlives_the_wait() -> None:
"""The 0.25 s wait after cancelling the response task may time out.
On Python 3.10 that timeout is ``asyncio.TimeoutError``, not the builtin
``TimeoutError``, and it must not stop the cancel from completing.
"""
async def outlives_the_first_cancel() -> None:
try:
await asyncio.sleep(5)
except asyncio.CancelledError:
await asyncio.sleep(1)
raise
h = await open_harness()
task = asyncio.create_task(outlives_the_first_cancel())
try:
await h.run(append_audio())
request_id = h.stage0_request_id()
await h.deliver_and_settle(tts_output(request_id, samples=24000, text="hello"))
h.runner.tasks.active_response_task = task
events = await h.run(commands.CancelResponse())
# The runner is still inside that wait when the mailbox goes quiet.
events += await h.settle(idle_s=0.6, timeout_s=5.0)
assert find(events, "response.done").status == "cancelled"
assert "error" not in types(events)
assert h.session.epoch == 1
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
await close_harness(h)
@pytest.mark.asyncio
async def test_listen_decision_is_consumed_and_never_forwarded_to_tts() -> None:
h = await open_harness()
@@ -505,6 +505,75 @@ class TestSpeechAPI:
assert voice_info["file_size"] == len(audio_content)
response = client.delete("/v1/audio/voices/test_voice")
def test_upload_voice_rejects_builtin_name(self, client):
"""A name colliding with a built-in or precomputed voice is rejected."""
handler = client.app.state.openai_serving_speech
handler._adapter.capabilities.supported_speakers = {"vivian"}
handler._adapter.capabilities.precomputed_speakers = {"ono_anna": {}}
files = {"audio_sample": ("test.wav", b"fake audio content" * 1000, "audio/wav")}
for reserved in ("Vivian", "ono_anna"):
response = client.post("/v1/audio/voices", files=files, data={"consent": "c1", "name": reserved})
assert response.status_code == 400
assert "reserved" in response.json()["detail"]
assert not handler.uploaded_speakers
def test_upload_voice_overwrites_same_name_by_default(self, client):
"""Default policy: re-registering an uploaded name replaces it in place."""
files = {"audio_sample": ("test.wav", b"first take", "audio/wav")}
data = {"consent": "c1", "name": "test_voice_ow"}
assert client.post("/v1/audio/voices", files=files, data=data).status_code == 200
files = {"audio_sample": ("test.wav", b"second take, longer content", "audio/wav")}
response = client.post("/v1/audio/voices", files=files, data=data)
assert response.status_code == 200
assert response.json()["voice"]["file_size"] == len(b"second take, longer content")
client.delete("/v1/audio/voices/test_voice_ow")
def test_upload_voice_immutable_policy_requires_delete(self, client):
"""VLLM_OMNI_SPEAKER_REGISTRATION_POLICY=immutable rejects duplicates until deleted."""
handler = client.app.state.openai_serving_speech
handler._registration_policy = "immutable"
try:
files = {"audio_sample": ("test.wav", b"fake audio content" * 1000, "audio/wav")}
data = {"consent": "c1", "name": "test_voice_imm"}
assert client.post("/v1/audio/voices", files=files, data=data).status_code == 200
response = client.post("/v1/audio/voices", files=files, data=data)
assert response.status_code == 400
assert "immutable" in response.json()["detail"]
assert client.delete("/v1/audio/voices/test_voice_imm").status_code == 200
assert client.post("/v1/audio/voices", files=files, data=data).status_code == 200
finally:
handler._registration_policy = "overwrite"
client.delete("/v1/audio/voices/test_voice_imm")
def test_invalid_registration_policy_fails_startup(self, mocker, monkeypatch, tmp_path):
"""An unknown VLLM_OMNI_SPEAKER_REGISTRATION_POLICY is a configuration error, not a silent fallback."""
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
monkeypatch.setenv("VLLM_OMNI_SPEAKER_REGISTRATION_POLICY", "append")
engine_client = mocker.MagicMock()
engine_client.default_sampling_params_list = [{}]
with pytest.raises(ValueError, match="VLLM_OMNI_SPEAKER_REGISTRATION_POLICY"):
OmniOpenAIServingSpeech(
engine_client=engine_client, models=mocker.MagicMock(), request_logger=mocker.MagicMock()
)
def test_restored_upload_shadowing_builtin_is_dropped(self, client):
"""A pre-guard upload restored under a built-in name must not keep shadowing it."""
handler = client.app.state.openai_serving_speech
handler._adapter.capabilities.supported_speakers = {"vivian"}
handler._adapter.capabilities.precomputed_speakers = {}
handler.uploaded_speakers["vivian"] = {"name": "vivian", "file_path": "/tmp/vivian.safetensors"}
handler.uploaded_speakers["keep_me"] = {"name": "keep_me", "file_path": "/tmp/keep_me.safetensors"}
handler._drop_shadowing_uploads()
assert "vivian" not in handler.uploaded_speakers
assert "keep_me" in handler.uploaded_speakers
handler.uploaded_speakers.pop("keep_me")
def test_upload_voice_with_ref_text(self, client, tmp_path):
"""Test voice upload with ref_text enables in-context cloning."""
audio_content = b"fake audio content" * 1000
@@ -33,9 +33,9 @@ def _make_omni(*, stage_types: list[str]) -> AsyncOmni:
omni._final_output_handler = lambda: None
omni.reset_mm_cache = AsyncMock()
stage_clients = [SimpleNamespace(stage_type=stage_type) for stage_type in stage_types]
stage_configs = [SimpleNamespace(stage_type=stage_type) for stage_type in stage_types]
omni.engine = SimpleNamespace(
stage_clients=stage_clients,
stage_configs=stage_configs,
stage_vllm_configs=[None] * len(stage_types),
collective_rpc_async=AsyncMock(return_value=[True]),
)
+31 -6
View File
@@ -19,7 +19,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm_omni.config.omni_config import VllmOmniConfig
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
from vllm_omni.config.stage_config import StageConfig
from vllm_omni.config.stage_config import StageConfig, StageType
from vllm_omni.engine.async_omni_engine import StageRuntimeInfo
from vllm_omni.engine.messages import ErrorMessage, OutputMessage
from vllm_omni.entrypoints.async_omni import AsyncOmni
@@ -107,7 +107,13 @@ class FakeAsyncOmniEngine:
self.stage_metadata = stage_metadata or [THREE_STAGE_META[-1]]
self.num_stages = len(self.stage_metadata)
self.stage_configs = [
StageConfig(stage_id=i, model_stage="dummy-model").to_omegaconf() for i in range(self.num_stages)
StageConfig(
stage_id=i,
model_stage="dummy-model",
stage_type=StageType(metadata.stage_type),
is_comprehension=getattr(metadata, "is_comprehension", False),
).to_omegaconf()
for i, metadata in enumerate(self.stage_metadata)
]
self.default_sampling_params_list = default_sampling_params_list or [
SamplingParams(max_tokens=8) for _ in range(self.num_stages)
@@ -481,7 +487,7 @@ def test_model_config_and_vllm_config_forward_from_comprehension_stage():
io_processor = SimpleNamespace(name="io-processor")
omni = object.__new__(AsyncOmni)
omni.engine = SimpleNamespace(
stage_clients=[SimpleNamespace(is_comprehension=False), SimpleNamespace(is_comprehension=True)],
stage_configs=[SimpleNamespace(is_comprehension=False), SimpleNamespace(is_comprehension=True)],
stage_vllm_configs=[None, vllm_config],
)
omni.input_processor = input_processor
@@ -502,7 +508,7 @@ def test_openai_serving_models_can_consume_async_omni_compat_attrs():
io_processor = SimpleNamespace(name="io-processor")
omni = object.__new__(AsyncOmni)
omni.engine = SimpleNamespace(
stage_clients=[SimpleNamespace(is_comprehension=True)],
stage_configs=[SimpleNamespace(is_comprehension=True)],
stage_vllm_configs=[vllm_config],
)
omni.input_processor = input_processor
@@ -524,10 +530,11 @@ def test_get_diffusion_od_config_returns_diffusion_stage_config():
diffusion_od_config = object()
omni = object.__new__(AsyncOmni)
omni.engine = SimpleNamespace(
stage_configs=[SimpleNamespace(stage_type="llm"), SimpleNamespace(stage_type="diffusion")],
stage_clients=[
SimpleNamespace(stage_type="llm"),
SimpleNamespace(stage_type="diffusion", od_config=diffusion_od_config),
]
],
)
assert omni.get_diffusion_od_config() is diffusion_od_config
@@ -537,15 +544,33 @@ def test_get_diffusion_od_config_falls_back_to_inner_engine():
diffusion_od_config = object()
omni = object.__new__(AsyncOmni)
omni.engine = SimpleNamespace(
stage_configs=[SimpleNamespace(stage_type="llm"), SimpleNamespace(stage_type="diffusion")],
stage_clients=[
SimpleNamespace(stage_type="llm"),
SimpleNamespace(stage_type="diffusion", _engine=SimpleNamespace(od_config=diffusion_od_config)),
]
],
)
assert omni.get_diffusion_od_config() is diffusion_od_config
@pytest.mark.asyncio
async def test_async_omni_rejects_diffusion_list_prompt_without_stage_clients(
monkeypatch: pytest.MonkeyPatch,
):
engine = FakeAsyncOmniEngine(stage_metadata=DIFFUSION_ONLY_META)
engine.stage_clients = []
_patch_engine(monkeypatch, engine)
app = AsyncOmni("dummy-model")
try:
with pytest.raises(ValueError, match="single prompt per request"):
async for _ in app.generate(prompt=["prompt-1", "prompt-2"], request_id="req-1"):
pass
finally:
app.shutdown()
@pytest.mark.asyncio
async def test_async_omni_yields_only_final_stage_outputs(monkeypatch: pytest.MonkeyPatch):
engine = FakeAsyncOmniEngine(
@@ -260,6 +260,37 @@ class TestDiTAttention:
assert attention.to_k.out_features == 512
assert attention.to_v.out_features == 512
def test_embedded_dit_honors_diffusion_attention_backend(self, monkeypatch):
"""The LLM-hosted code2wav DiT must consume the diffusion env override."""
from vllm_omni.diffusion.config import get_current_diffusion_config_or_none
from vllm_omni.model_executor.models.cosyvoice3.cosyvoice3_code2wav import _build_dit_estimator
monkeypatch.setenv("DIFFUSION_ATTENTION_BACKEND", "TORCH_SDPA")
estimator = _build_dit_estimator(
{
"dim": 32,
"depth": 1,
"heads": 2,
"dim_head": 16,
"dropout": 0.0,
"ff_mult": 2,
"mel_dim": 8,
"mu_dim": 8,
"spk_dim": 4,
"out_channels": 8,
"static_chunk_size": 4,
"num_decoding_left_chunks": 1,
}
)
attention = estimator.transformer_blocks[0].attn.attn
assert attention.backend_pref == "TORCH_SDPA"
assert attention.backend_explicit is True
assert attention.attn_backend is not None
assert attention.attn_backend.get_name() == "SDPA"
assert get_current_diffusion_config_or_none() is None
@pytest.mark.core_model
@pytest.mark.cpu
def test_forward_passes_2d_mask_without_casting_qkv(self):
@@ -39,7 +39,10 @@ CHUNK_LEN = 24
TOTAL_MEL = 96
SPM = 480 # samples per mel frame for the small test HiFT (8*5*3*4), matching the real model
LONG_TOTAL_MEL = 400 # long enough that the 64-frame window actually truncates
WINDOW_LEN = 64
# Six chunks exercise three updates after history exceeds the bounded window,
# without spending minutes recomputing an unnecessarily long synthetic signal.
LONG_TOTAL_MEL = 144
PCM16_LSB = 1.0 / 32767.0 # smallest representable PCM16 difference
# Worst observed deviation is 1.97e-7 (~2x float32 eps); atol=1e-6 keeps 5x headroom.
@@ -189,16 +192,19 @@ def test_incremental_hift_bounded_window_is_close(config):
def test_incremental_hift_matches_streaming_reference_within_tolerance(config):
"""Windowed output matches the full-cumulative reference within float32 rounding.
The load-bearing correctness test: 400 mel frames far exceeds the 64-frame
window, so truncation genuinely exercises the phase/noise carry. Float64
agrees to ~1e-16, confirming the residual here is rounding, not truncation.
The load-bearing correctness test crosses the 64-frame window repeatedly,
so truncation genuinely exercises the phase/noise carry. Float64 agrees to
~1e-16, confirming the residual here is rounding, not truncation.
"""
hift = _make_hift(config)
model = _make_model(hift, window_len=64) # the real _hift_window_len
model = _make_model(hift, window_len=WINDOW_LEN) # the real _hift_window_len
chunks = _chunks(total_mel=LONG_TOTAL_MEL)
trim = int(hift.f0_predictor.condnet[0].causal_padding)
assert LONG_TOTAL_MEL > 64 + trim + CHUNK_LEN # window must actually truncate
truncating_steps = sum(
history_len > WINDOW_LEN + trim for history_len in range(CHUNK_LEN, LONG_TOTAL_MEL, CHUNK_LEN)
)
assert truncating_steps >= 3 # exercise repeated truncation, not only the boundary
full = _full_reference(model, chunks)
incr = _incremental(model, chunks)
@@ -0,0 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""VLLM_OMNI_VOXCPM_CODE_PATH still works but warns that it is deprecated (#6232)."""
from pathlib import Path
import pytest
from vllm_omni.model_executor.models.voxcpm2 import voxcpm2_import_utils
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _capture_warnings(monkeypatch) -> list[str]:
messages: list[str] = []
monkeypatch.setattr(voxcpm2_import_utils.logger, "warning_once", lambda msg, *args: messages.append(msg))
return messages
def test_code_path_env_warns_and_is_still_used(monkeypatch, tmp_path: Path):
messages = _capture_warnings(monkeypatch)
monkeypatch.setenv("VLLM_OMNI_VOXCPM_CODE_PATH", str(tmp_path))
candidates = voxcpm2_import_utils._iter_voxcpm2_src_candidates()
assert candidates[0] == tmp_path
assert len(messages) == 1
assert "VLLM_OMNI_VOXCPM_CODE_PATH is deprecated" in messages[0]
def test_no_warning_without_code_path_env(monkeypatch):
messages = _capture_warnings(monkeypatch)
monkeypatch.delenv("VLLM_OMNI_VOXCPM_CODE_PATH", raising=False)
voxcpm2_import_utils._iter_voxcpm2_src_candidates()
assert messages == []
@@ -142,6 +142,37 @@ def test_native_duplex_speak_segment_reaches_split_talker() -> None:
assert info["duplex"]["turn_id"] == 7
@pytest.mark.parametrize("folded_decisions", [0, 1, 2])
@pytest.mark.parametrize("bos_in_prompt", [False, True])
def test_native_duplex_tts_bos_aligns_after_window_rebuild(folded_decisions, bos_in_prompt) -> None:
prompt_ids = [101] * 10
output_ids = [9303] * folded_decisions + [9301, 21, 22, 9308]
if bos_in_prompt:
prompt_ids[-1] = 9301
output_ids = [21, 22, 9308]
rows = len(prompt_ids) + len(output_ids) - folded_decisions
latent = torch.arange(rows * 4, dtype=torch.float32).reshape(rows, 4)
source = _output(
prompt_ids=prompt_ids,
output_ids=output_ids,
latent=latent,
multimodal_output={
"duplex_prompt_token_ids": prompt_ids,
"meta": {
"tts_bos_token_id": 9301,
"tts_eos_token_id": 9302,
"listen_token_id": 9303,
"speak_token_id": 9304,
"chunk_eos_token_id": 9308,
},
},
)
converted = llm2tts([source], prompt=[{}], _streaming_context=SimpleNamespace(bridge_states={}))[0]
info = converted["model_intermediate_buffer"]
assert info["ids"]["tts"] == [21, 22]
torch.testing.assert_close(torch.as_tensor(info["hidden_states"]["tts"]), latent[-3:-1])
def test_native_duplex_continuation_appends_only_new_talker_condition() -> None:
prompt_ids = [101, 102]
token_ids = {
+170
View File
@@ -848,6 +848,176 @@ def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format():
assert result["prompt_suffix_len"] == 0
@pytest.mark.parametrize("mode", [None, "off", "basic", "context"])
def test_minicpmo_stage0_history_collection_starts_with_session_config(mode):
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import _MiniCPMO45Stage0SessionState
runtime = _stage0_vision_runtime()
state = _MiniCPMO45Stage0SessionState(session_id="history-lifecycle")
config = {} if mode is None else {"duplex_window_config": {"sliding_window_mode": mode}}
runtime._prepare_session_context(state, {}, runtime_config=config)
for seq in range(1, 101):
runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=seq)
enabled = mode in {"basic", "context"}
assert state.window_enabled is enabled
assert len(state.window_units) == (99 if enabled else 0)
assert (state.pending_window_unit is not None) is enabled
assert state.pending_window_generated_tokens == []
@pytest.mark.parametrize("mode", ["basic", "context"])
@pytest.mark.parametrize("buffered", [False, True])
def test_minicpmo_stage0_multi_chunk_append_matches_scheduler_window(mode, buffered):
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import _MiniCPMO45Stage0SessionState
runtime = _stage0_vision_runtime()
state = _MiniCPMO45Stage0SessionState(session_id="multi-chunk-window", window_enabled=True)
if buffered:
# Keep the tiny fake processor's initial chunk unpadded to model carry.
runtime._pad_first_audio_chunk_if_needed = lambda *args: None
assert runtime._stage_prefill_embeddings_only(state, np.zeros(2, dtype=np.float32), seq=0)["success"] is False
first = runtime._stage_prefill_embeddings_only(
state, np.zeros(6 if buffered else 8, dtype=np.float32), seq=1, final=buffered
)
assert first["input_token_ids"] == [1, 11, 2, 1, 11]
assert state.window_units == []
assert state.pending_window_unit.token_ids == first["input_token_ids"]
session = SimpleNamespace(num_prompt_tokens=5)
replaced = False
for seq in (2, 3):
info = {
"duplex": {
"data_plane": True,
"seq": seq,
"runtime_config": {
"duplex_window_config": {
"sliding_window_mode": mode,
"basic_window_high_tokens": 1,
"basic_window_low_tokens": 2,
"context_max_units": 1,
"context_previous_max_tokens": 16,
},
"duplex_window_previous_marker_token_ids": [201, 5],
},
}
}
update = SimpleNamespace(prompt_token_ids=[3, 2, 1, 11], model_intermediate_buffer=info)
planned = OmniARScheduler._prepare_minicpmo45_stage0_window(
session, update, segment_output_ids=[44], completed_terminator=3
)
result = runtime._stage_prefill_embeddings_only(
state, np.zeros(4, dtype=np.float32), seq=seq, stage0_window=info["duplex"]["stage0_window"]
)
assert result["stage0_window_replaced"] is planned
if planned:
assert result["num_input_tokens"] == len(update.prompt_token_ids)
replaced = True
break
session.num_prompt_tokens += 1 + len(update.prompt_token_ids)
assert replaced
def test_minicpmo_stage0_basic_window_rebuilds_from_retained_units():
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
runtime = _stage0_vision_runtime()
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-basic-window", window_enabled=True)
first = runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
assert first["input_token_ids"] == [1, 11]
state.pending_window_generated_tokens.append(5)
state.pending_terminator_token = 3
rebuilt = runtime._stage_prefill_embeddings_only(
state,
np.zeros(4, dtype=np.float32),
seq=2,
stage0_window={
"replace": True,
"mode": "basic",
"drop_units": 1,
"replacement_prompt_len": 2,
},
)
assert rebuilt["stage0_window_replaced"] is True
assert rebuilt["input_token_ids"] == [1, 11]
assert state.window_units == []
def test_minicpmo_stage0_context_window_inserts_previous_before_suffix():
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
runtime = _stage0_vision_runtime()
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-context-window", window_enabled=True)
prefix = runtime._embed_token(200)
suffix = runtime._embed_token(202)
state.context_embeds = [prefix, suffix]
state.context_token_ids = [200, 202]
state.context_prefix_embeds = [prefix]
state.context_prefix_token_ids = [200]
state.context_suffix_embeds = [suffix]
state.context_suffix_token_ids = [202]
first = runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
assert first["input_token_ids"] == [200, 202, 1, 11]
state.pending_window_generated_tokens.append(42)
state.pending_terminator_token = 3
rebuilt = runtime._stage_prefill_embeddings_only(
state,
np.zeros(4, dtype=np.float32),
seq=2,
stage0_window={
"replace": True,
"mode": "context",
"drop_units": 1,
"previous_token_ids": [42],
"previous_marker_token_ids": [201, 5],
"replacement_prompt_len": 7,
},
)
assert rebuilt["input_token_ids"] == [200, 201, 5, 42, 202, 1, 11]
assert rebuilt["num_input_tokens"] == 7
@pytest.mark.parametrize("pending_terminator", [None, 3, 99])
def test_minicpmo_stage0_window_uses_accepted_output_not_async_sampler_history(pending_terminator):
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
runtime = _stage0_vision_runtime()
runtime._stage_audio_embeddings = lambda *args, **kwargs: torch.zeros((10, 2))
state = _MiniCPMO45Stage0SessionState(session_id="sid-accepted-window", window_enabled=True)
state.context_prefix_token_ids = [200] * 7
state.context_suffix_token_ids = [202]
state.context_token_ids = [*state.context_prefix_token_ids, 202]
state.context_prefix_embeds = [runtime._embed_token(token_id) for token_id in state.context_prefix_token_ids]
state.context_suffix_embeds = [runtime._embed_token(202)]
state.context_embeds = [*state.context_prefix_embeds, *state.context_suffix_embeds]
runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
for seq in (2, 3):
state.pending_window_generated_tokens = [3, 40, 41]
state.pending_terminator_token = pending_terminator
plan = {"completed_token_ids": [], "completed_terminator_token_id": 3}
if seq == 3:
plan.update(replace=True, mode="context", drop_units=1, replacement_prompt_len=32)
result = runtime._stage_prefill_embeddings_only(
state, np.zeros(4, dtype=np.float32), seq=seq, stage0_window=plan
)
assert result["num_input_tokens"] == 32
assert [len(unit.token_ids) for unit in state.window_units] == [13]
assert len(state.pending_window_unit.token_ids) == 11
assert 40 not in result["input_token_ids"]
assert 41 not in result["input_token_ids"]
def _stage0_vision_runtime():
import torch