Merge branch 'main' into fix/tp-local-kv-recv-consensus

This commit is contained in:
Gao Han authored and GitHub committed 2026-08-01 23:23:52 +08:00
commit 62aeffd15c
61 files changed
+1881 -224

No files matched your search

@@ -0,0 +1,249 @@
"""Tests for Daily-Omni ``--daily-omni-pack-mode`` packing recipes.
The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
OpenBMB's reported Daily-Omni score when frames and 1s audio segments arrive strictly
interleaved, with ``max_slice_nums=1`` / ``use_image_id=False`` and an empty MiniCPM
system message (not the Qwen system prompt). These tests pin that contract so a refactor
cannot silently regress the score.
vllm stubs are installed by tests/benchmarks/conftest.py before collection.
"""
from __future__ import annotations
import importlib.util
import json
import sys
from pathlib import Path
import numpy as np
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
_REPO_ROOT = Path(__file__).resolve().parents[2]
_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "daily_omni_dataset.py"
_MODULE_NAME = "vllm_omni.benchmarks.data_modules.daily_omni_dataset"
if _MODULE_NAME not in sys.modules:
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
_mod = importlib.util.module_from_spec(_spec)
sys.modules[_MODULE_NAME] = _mod
_spec.loader.exec_module(_mod)
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( # noqa: E402
DailyOmniDataset,
_uniform_sample_indices,
)
_VIDEO_ID = "vid001"
_NUM_FRAMES = 5
@pytest.fixture
def video_dir(tmp_path: Path) -> Path:
"""Official Daily-Omni ``Videos/`` layout with placeholder media files."""
per_video = tmp_path / "Videos" / _VIDEO_ID
per_video.mkdir(parents=True)
(per_video / f"{_VIDEO_ID}_video.mp4").write_bytes(b"fake-mp4")
(per_video / f"{_VIDEO_ID}_audio.wav").write_bytes(b"fake-wav")
return tmp_path / "Videos"
@pytest.fixture
def qa_json(tmp_path: Path) -> Path:
path = tmp_path / "qa.json"
path.write_text(
json.dumps(
[
{
"video_id": _VIDEO_ID,
"question": "What happens first?",
"choice": {"A": "a", "B": "b", "C": "c", "D": "d"},
"answer": "A",
"task_type": "AV Event Alignment",
"video_duration": "30s",
}
]
),
encoding="utf-8",
)
return path
def _make_dataset(qa_json: Path, video_dir: Path, **kwargs) -> DailyOmniDataset:
return DailyOmniDataset(
qa_json_path=str(qa_json),
video_dir=str(video_dir),
disable_shuffle=True,
**kwargs,
)
def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) -> None:
"""Replace decord/ffmpeg extraction with deterministic in-memory frames + segments."""
from PIL import Image
def fake_extract(video_path, *, audio_path, include_audio, **_kw):
counter.append(1)
frames = [Image.new("RGB", (16, 16), color=(i, i, i)) for i in range(_NUM_FRAMES)]
segments = [np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
return frames, segments
monkeypatch.setattr(dataset, "_extract_minicpm_frame_audio_segments", fake_extract)
# ---------------------------------------------------------------------------
# _uniform_sample_indices
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(("n", "k"), [(10, 4), (100, 64), (1000, 64), (7, 7), (3, 5), (1, 1)])
def test_uniform_sample_indices_matches_openbmb_midpoint(n: int, k: int) -> None:
"""Pin OpenBMB midpoint-bin sampling (not np.linspace endpoints)."""
got = _uniform_sample_indices(n, k)
if n <= k:
assert got == list(range(n))
else:
gap = n / k
expected = [int(i * gap + gap / 2) for i in range(k)]
assert got == expected
# Endpoints differ from linspace for typical Daily-Omni downsample sizes.
linspace = [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
if n == 1000 and k == 64:
assert got[0] == 7 and got[-1] == 992
assert linspace[0] == 0 and linspace[-1] == 999
assert got != linspace
def test_uniform_sample_indices_non_positive_k() -> None:
assert _uniform_sample_indices(10, 0) == []
assert _uniform_sample_indices(10, -1) == []
# ---------------------------------------------------------------------------
# minicpm-interleave packing
# ---------------------------------------------------------------------------
def test_minicpm_interleave_alternates_image_and_audio(monkeypatch, qa_json, video_dir) -> None:
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
_stub_extract(monkeypatch, ds, [])
parts, extra, position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
assert position == "first"
assert [p["type"] for p in parts] == ["image_url", "audio_url"] * _NUM_FRAMES
assert extra["mm_processor_kwargs"] == {
"use_audio_in_video": False,
"max_slice_nums": 1,
"use_image_id": False,
}
def test_minicpm_interleave_visual_mode_emits_frames_only(monkeypatch, qa_json, video_dir) -> None:
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="visual")
_stub_extract(monkeypatch, ds, [])
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
assert [p["type"] for p in parts] == ["image_url"] * _NUM_FRAMES
def test_minicpm_interleave_writes_file_urls_under_video_dir(monkeypatch, qa_json, video_dir) -> None:
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
_stub_extract(monkeypatch, ds, [])
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
segment_dir = video_dir / ".minicpm_daily_omni_interleave" / _VIDEO_ID
for part in parts:
url = part[part["type"]]["url"]
assert url.startswith("file://")
assert Path(url[len("file://") :]).parent == segment_dir.resolve()
assert sorted(p.name for p in segment_dir.glob("frame_*.jpg")) == [f"frame_{i:04d}.jpg" for i in range(_NUM_FRAMES)]
def test_minicpm_interleave_inline_mode_emits_data_urls(monkeypatch, qa_json, video_dir) -> None:
ds = _make_dataset(
qa_json,
video_dir,
pack_mode="minicpm-interleave",
input_mode="all",
inline_local_video=True,
)
_stub_extract(monkeypatch, ds, [])
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
assert parts[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
assert parts[1]["audio_url"]["url"].startswith("data:audio/wav;base64,")
assert not (video_dir / ".minicpm_daily_omni_interleave").exists()
def test_minicpm_interleave_caches_file_parts_but_not_inline_payloads(monkeypatch, qa_json, video_dir) -> None:
"""Inline base64 must not be retained: ~700 Daily-Omni videos would pin GBs of RSS."""
file_calls: list[int] = []
ds_file = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
_stub_extract(monkeypatch, ds_file, file_calls)
ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
assert len(file_calls) == 1
assert ds_file._minicpm_interleave_cache
inline_calls: list[int] = []
ds_inline = _make_dataset(
qa_json,
video_dir,
pack_mode="minicpm-interleave",
input_mode="all",
inline_local_video=True,
)
_stub_extract(monkeypatch, ds_inline, inline_calls)
ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
assert len(inline_calls) == 2
assert not ds_inline._minicpm_interleave_cache
def test_minicpm_interleave_missing_local_video_returns_none(qa_json, tmp_path) -> None:
empty = tmp_path / "empty_videos"
empty.mkdir()
ds = _make_dataset(qa_json, empty, pack_mode="minicpm-interleave", input_mode="all")
parts, extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
assert parts is None
assert extra is None
# ---------------------------------------------------------------------------
# System prompt selection
# ---------------------------------------------------------------------------
def test_minicpm_interleave_keeps_empty_system_message(monkeypatch, qa_json, video_dir) -> None:
"""Official MiniCPM omni mode still sends role=system with empty content."""
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
_stub_extract(monkeypatch, ds, [])
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
messages = ds._build_daily_omni_openai_messages(parts, "q?", {"A": "a"})
assert [m["role"] for m in messages] == ["system", "user"]
assert messages[0]["content"][0]["text"] == ""
def test_qwen_pack_mode_keeps_system_message(qa_json, video_dir) -> None:
ds = _make_dataset(qa_json, video_dir, pack_mode="qwen", input_mode="all")
messages = ds._build_daily_omni_openai_messages([], "q?", {"A": "a"})
assert [m["role"] for m in messages] == ["system", "user"]
assert "Qwen" in messages[0]["content"][0]["text"]
def test_invalid_pack_mode_rejected(qa_json, video_dir) -> None:
with pytest.raises(ValueError, match="pack_mode must be"):
_make_dataset(qa_json, video_dir, pack_mode="bogus")
+26
View File
@@ -2214,6 +2214,32 @@ class TestPlatformOverrides:
assert rocm.stages[0].enforce_eager is None
assert rocm.stages[1].enforce_eager is True
def test_minicpmo_4_5_cuda_caps_talker_kv_cache(self):
pipeline = resolve_pipeline_config("minicpmo_4_5")
assert isinstance(pipeline, PipelineConfig)
deploy_path = Path(get_deploy_config_path("minicpmo_4_5.yaml"))
cuda = _apply_platform_overrides(load_deploy_config(deploy_path), platform="cuda")
cuda_stages = merge_pipeline_deploy(pipeline, cuda)
assert cuda_stages[1].yaml_engine_args["kv_cache_memory_bytes"] == 2 * 1024**3
# The CUDA memory budget must not leak into the existing NPU profile.
npu = _apply_platform_overrides(load_deploy_config(deploy_path), platform="npu")
npu_stages = merge_pipeline_deploy(pipeline, npu)
assert "kv_cache_memory_bytes" not in npu_stages[1].yaml_engine_args
# Multi-GPU replica profiles place Talker on dedicated devices and
# preserve their previous automatic KV-cache sizing.
for filename in (
"minicpmo_4_5_3gpu_stage1_replicas.yaml",
"minicpmo_4_5_4gpu_stage1_replicas.yaml",
):
replica = _apply_platform_overrides(
load_deploy_config(Path(get_deploy_config_path(filename))), platform="cuda"
)
replica_stages = merge_pipeline_deploy(pipeline, replica)
assert replica_stages[1].yaml_engine_args["kv_cache_memory_bytes"] is None
def test_npu_overrides(self):
deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml"))
if not deploy_path.exists():
@@ -0,0 +1,57 @@
from types import SimpleNamespace
import pytest
from vllm import SamplingParams
from vllm.v1.engine import EngineCoreRequest
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
from vllm_omni.engine.async_engine_utils import apply_omni_final_stage_metadata
from vllm_omni.engine.serialization import deserialize_additional_information
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_engine_request() -> EngineCoreRequest:
return EngineCoreRequest(
request_id="req",
prompt_token_ids=[1],
mm_features=None,
sampling_params=SamplingParams(max_tokens=1),
pooling_params=None,
arrival_time=0.0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
def _request_omits_kv_transfer(*, force_kv_transfer: bool) -> tuple[bool, dict]:
tagged = apply_omni_final_stage_metadata(
_make_engine_request(),
final_stage_id=0,
force_kv_transfer=force_kv_transfer,
)
scheduler = OmniARScheduler.__new__(OmniARScheduler)
scheduler._omits_kv_transfer_cache = {}
request = SimpleNamespace(
request_id="req",
additional_information=tagged.additional_information,
)
result = scheduler._request_omits_kv_transfer_to_next_stage(request)
metadata = deserialize_additional_information(tagged.additional_information)
return result, metadata
def test_stage_zero_request_omits_kv_transfer():
omits_transfer, metadata = _request_omits_kv_transfer(force_kv_transfer=False)
assert omits_transfer
assert "omni_force_kv_transfer" not in metadata
def test_cfg_companion_forces_kv_transfer_without_downstream_payload():
omits_transfer, metadata = _request_omits_kv_transfer(force_kv_transfer=True)
assert not omits_transfer
assert metadata["omni_final_stage_id"] == 0
assert metadata["omni_force_kv_transfer"] is True
@@ -21,6 +21,7 @@ import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.engine import EngineCoreEventType
from vllm.v1.metrics.stats import PrefillStats
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
@@ -93,6 +94,17 @@ def _make_update(**kwargs) -> StreamingUpdate:
return StreamingUpdate(**defaults)
def test_generation_scheduler_records_prefill_stats_for_metrics() -> None:
request = _make_request(prompt_token_ids=[1, 2, 3, 4])
request.prefill_stats = PrefillStats()
OmniGenerationScheduler._record_prefill_stats(request)
assert request.prefill_stats.num_prompt_tokens == 4
assert request.prefill_stats.num_computed_tokens == 4
assert request.prefill_stats.num_cached_tokens == 0
def test_resumable_generation_stop_marks_segment_boundary() -> None:
session = _make_request(request_id="req-generation-segment")
session.status = RequestStatus.RUNNING
@@ -9,7 +9,9 @@ import torch
from torch import nn
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import Wan22Pipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@@ -91,7 +93,20 @@ def _make_pipeline() -> Wan22Pipeline:
return pipeline
def test_forward_delegates_denoising_to_diffuse() -> None:
@pytest.mark.parametrize(
("sampling_params_kwargs", "expected_low", "expected_high"),
[
({}, 4.0, 4.0),
({"guidance_scale": 0.0}, 0.0, 0.0),
({"guidance_scale": 1.0}, 1.0, 1.0),
({"guidance_scale": 3.0, "guidance_scale_2": 5.0}, 3.0, 5.0),
],
)
def test_forward_delegates_denoising_to_diffuse(
sampling_params_kwargs: dict[str, float],
expected_low: float,
expected_high: float,
) -> None:
pipeline = _make_pipeline()
captured: dict[str, object] = {}
@@ -101,25 +116,15 @@ def test_forward_delegates_denoising_to_diffuse() -> None:
pipeline.diffuse = _fake_diffuse # type: ignore[method-assign]
mock_req = SimpleNamespace(
mock_req = OmniDiffusionRequest(
prompt="prompt",
request_id="test-req",
sampling_params=SimpleNamespace(
height=None,
width=None,
sampling_params=OmniDiffusionSamplingParams(
num_frames=1,
num_inference_steps=2,
guidance_scale_provided=True,
guidance_scale=1.0,
guidance_scale_2=None,
boundary_ratio=None,
generator=None,
seed=None,
num_outputs_per_prompt=1,
max_sequence_length=32,
latents=None,
output_type="latent",
extra_args={},
**sampling_params_kwargs,
),
)
batch = DiffusionRequestBatch(requests=[mock_req])
@@ -129,8 +134,8 @@ def test_forward_delegates_denoising_to_diffuse() -> None:
assert torch.equal(output.output, torch.ones((1, 4, 1, 8, 8)))
assert torch.equal(captured["prompt_embeds"], torch.zeros(1, 32, 8))
assert torch.equal(captured["timesteps"], pipeline.scheduler.timesteps)
assert captured["guidance_low"] == 1.0
assert captured["guidance_high"] == 1.0
assert captured["guidance_low"] == expected_low
assert captured["guidance_high"] == expected_high
assert captured["boundary_timestep"] == pytest.approx(875.0)
assert captured["latent_condition"] is None
assert captured["first_frame_mask"] is None
+37
View File
@@ -72,3 +72,40 @@ def test_tp_seed_same_across_ranks_and_varies_across_requests():
# Seeds must vary across requests (non-determinism preserved).
assert len(set(seeds)) == n_requests, f"Expected {n_requests} unique seeds but got {len(set(seeds))}: {seeds}"
def _request_with_guidance(**sampling_kwargs) -> OmniDiffusionRequest:
return OmniDiffusionRequest(
prompt={"prompt": "a cup of coffee on a table"},
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1, **sampling_kwargs),
request_id="request-test",
)
def test_omitted_guidance_scale_is_unset_and_resolved_to_one():
# No guidance_scale passed: the caller did not set it, so it must NOT count as
# "provided" (each pipeline applies its own default), and __post_init__ resolves the
# None "unset" sentinel to a concrete float so no consumer ever observes None.
req = _request_with_guidance()
assert req.sampling_params.guidance_scale_provided is False
assert req.sampling_params.guidance_scale == 1.0
def test_explicit_zero_guidance_scale_is_honored():
# Regression (#4998): an explicit guidance_scale=0 (disable CFG) must be honored,
# not mistaken for "unset" and overridden by the pipeline's model default. This is
# the sentinel collision that the None default + ``is not None`` check fixes.
req = _request_with_guidance(guidance_scale=0.0)
assert req.sampling_params.guidance_scale_provided is True
assert req.sampling_params.guidance_scale == 0.0
@pytest.mark.parametrize("value", [0.999, 1.0, 5.0])
def test_explicit_nonzero_guidance_scale_is_honored(value):
# Any explicitly-set value (truthy or not) is honored as provided and preserved.
req = _request_with_guidance(guidance_scale=value)
assert req.sampling_params.guidance_scale_provided is True
assert req.sampling_params.guidance_scale == value
@@ -10,8 +10,10 @@ import pytest
import torch
from pytest_mock import MockerFixture
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
from vllm.v1.metrics.stats import PrefillStats, PromptTokenStats
from vllm.v1.request import RequestStatus
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
from vllm_omni.data_entry_keys import CodesStruct, MetaStruct, OmniPayload, OmniPayloadStruct
from vllm_omni.distributed.omni_connectors.adapter import construct_next_stage_streaming_input_prompt
from vllm_omni.distributed.omni_connectors.transfer_adapter.base import OmniTransferAdapterBase
@@ -37,8 +39,10 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None)
external_req_id=external_req_id or req_id,
status=status,
prompt_token_ids=[],
num_prompt_tokens=0,
num_computed_tokens=0,
num_output_placeholders=0,
prefill_stats=None,
additional_information=None,
is_finished=lambda: status == RequestStatus.FINISHED_STOPPED,
)
@@ -485,6 +489,48 @@ def test_load_poll_ar_request_additional_information_concats_tensors(build_adapt
assert request.additional_information["meta"]["finished"].item() is True
def test_non_ar_poll_reinitializes_prefill_stats_for_later_chunks(build_adapter):
adapter, connector = build_adapter(stage_id=2, model_mode="generation")
request = _req("req-later-chunk", RequestStatus.WAITING, external_req_id="ext-later-chunk")
request.prefill_stats = PrefillStats()
adapter.request_ids_mapping[request.request_id] = request.external_req_id
connector.get.return_value = (
{
"codes": {"audio": torch.tensor([7, 8], dtype=torch.long)},
"meta": {"finished": torch.tensor(False, dtype=torch.bool)},
},
8,
)
assert adapter._poll_single_request(request) is True
OmniGenerationScheduler._record_prefill_stats(request)
first_chunk_stats = request.prefill_stats
request.prefill_stats = None
connector.get.return_value = (
{
"codes": {"audio": torch.tensor([9, 10, 11], dtype=torch.long)},
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
},
8,
)
assert adapter._poll_single_request(request) is True
assert isinstance(request.prefill_stats, PrefillStats)
OmniGenerationScheduler._record_prefill_stats(request)
second_chunk_stats = request.prefill_stats
assert first_chunk_stats is not None
assert first_chunk_stats.num_prompt_tokens == 2
assert second_chunk_stats is not None
assert second_chunk_stats.num_prompt_tokens == 3
prompt_token_stats = PromptTokenStats()
prompt_token_stats.update_from_output(first_chunk_stats)
prompt_token_stats.update_from_output(second_chunk_stats)
assert prompt_token_stats.total == 5
assert prompt_token_stats.computed == 5
def test_sender_only_adapter_does_not_park_or_clear_requests(build_adapter):
adapter, _ = build_adapter(stage_id=1, connector_extra={"role": "sender"})
request = _req("req-1", RequestStatus.WAITING)
@@ -468,6 +468,10 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None:
"messages": messages,
"stream": True,
"key_words": {"text": ["london"]},
# A one-word English clip is too short for whisper's language detection to
# be stable; unpinned it renders "London" as 런던 / ランデ / 梁敦 and the
# containment check then fails on correct audio.
"transcript_language": "en",
}
# Retry only when assert_omni_response fails on text/audio cosine similarity (see tests/helpers/assertions.py).
@@ -7,7 +7,9 @@ from vllm.v1.engine import EngineCoreRequest
from vllm_omni.distributed.omni_coordinator import ReplicaInfo, ReplicaStatus
from vllm_omni.engine import OmniEngineCoreRequest
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine, StageRuntimeInfo
from vllm_omni.engine.serialization import deserialize_additional_information
from vllm_omni.engine.stage_pool import StagePool
from vllm_omni.model_executor.stage_input_processors.bagel import ExpandedPrompt
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -107,6 +109,34 @@ def test_build_add_request_message_preserves_model_intermediate_buffer(mocker: M
assert torch.equal(info["hidden_states"]["tts"], hidden)
def test_cfg_companion_suppresses_payload_but_forces_kv_transfer(mocker: MockerFixture):
engine = object.__new__(AsyncOmniEngine)
params = SamplingParams(max_tokens=8)
engine.prompt_expand_func = lambda *_args: [
ExpandedPrompt(
prompt={"prompt": "negative"},
role="cfg_text",
request_id_suffix="__cfg_text",
)
]
engine.supported_tasks = ("generate",)
engine.input_processor = mocker.Mock()
engine.input_processor.process_inputs.return_value = _make_engine_core_request("req__cfg_text")
engine.request_queue = mocker.Mock()
engine._enqueue_cfg_companions(
parent_id="req",
original_prompt={"prompt": "positive"},
stage0_params=params,
sampling_params_list=[params],
)
message = engine.request_queue.sync_q.put.call_args.args[0]
metadata = deserialize_additional_information(message.prompt.additional_information)
assert metadata["omni_final_stage_id"] == 0
assert metadata["omni_force_kv_transfer"] is True
def test_build_add_request_message_with_resumable_streaming(mocker: MockerFixture):
engine = object.__new__(AsyncOmniEngine)
params = SamplingParams(max_tokens=8)
@@ -172,6 +172,41 @@ def test_stage_engine_core_client_module_reload_keeps_forward_refs_deferred():
)
def test_async_omni_engine_initialize_stages_passes_log_stats_to_runtime(monkeypatch):
import vllm_omni.engine.async_omni_engine as engine_mod
engine = object.__new__(AsyncOmniEngine)
engine.stage_configs = [types.SimpleNamespace()]
engine.model = "dummy-model"
engine.config_path = "dummy-config"
engine.single_stage_mode = False
engine.diffusion_batch_size = 1
engine.async_chunk = False
engine.tokenizer = None
engine._single_stage_id_filter = None
engine._omni_master_address = None
engine._omni_master_port = None
engine._omni_dp_size_local = 1
engine._omni_heartbeat_timeout = 30.0
engine._omni_lb_policy = "random"
engine.request_queue = types.SimpleNamespace()
engine._log_stats = True
captured: dict[str, object] = {}
runtime = types.SimpleNamespace(stage_pools=[], initialize=lambda: None)
def _capture_create_stage_runtime(**kwargs):
captured.update(kwargs)
return runtime
monkeypatch.setattr(engine_mod, "create_stage_runtime", _capture_create_stage_runtime)
engine._initialize_stages(stage_init_timeout=7)
assert captured["stage_init_timeout"] == 7
assert captured["log_stats"] is True
def test_compute_replica_layout_splits_diffusion_devices_by_world_size():
stage_cfg = types.SimpleNamespace(
stage_id=0,
@@ -342,6 +377,95 @@ def test_stage_runtime_initializes_stage_pools(monkeypatch):
assert runtime.stage_pools[1].output_processor is stage1_output_processor
def test_stage_runtime_passes_log_stats_to_llm_replica_launch(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
runtime = StageRuntime(
stage_configs=[types.SimpleNamespace()],
model="dummy-model",
config_path="dummy-config",
stage_init_timeout=1,
diffusion_batch_size=1,
async_chunk=False,
log_stats=True,
)
cfg = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
plan = _make_llm_plan(0, stage_id=0, vllm_config=cfg).replicas[0]
plan.engine_args_dict = {}
captured: dict[str, object] = {}
addresses = types.SimpleNamespace(
inputs=["tcp://127.0.0.1:1"],
outputs=["tcp://127.0.0.1:2"],
frontend_stats_publish_address=None,
)
resources = types.SimpleNamespace(manager=object(), coordinator=None, addresses=addresses)
stage_client = types.SimpleNamespace()
@contextlib.contextmanager
def _capture_launch_stage_replica(**kwargs):
captured["log_stats"] = kwargs["log_stats"]
yield resources
monkeypatch.setattr(runtime_mod, "acquire_device_locks", lambda *_args, **_kwargs: [])
monkeypatch.setattr(runtime_mod, "launch_stage_replica", _capture_launch_stage_replica)
monkeypatch.setattr(
runtime_mod.StageEngineCoreClientBase,
"make_async_mp_client",
lambda **kwargs: (captured.__setitem__("client_log_stats", kwargs["log_stats"]) or stage_client),
)
assert runtime._initialize_local_llm_replica(plan, stage_init_timeout=1) is stage_client
assert captured["log_stats"] is True
assert captured["client_log_stats"] is True
def test_stage_runtime_passes_log_stats_to_output_processor(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
runtime = StageRuntime(
stage_configs=[types.SimpleNamespace()],
model="dummy-model",
config_path="dummy-config",
stage_init_timeout=1,
diffusion_batch_size=1,
async_chunk=False,
log_stats=True,
)
cfg = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
stage_plan = _make_llm_plan(0, stage_id=0, vllm_config=cfg)
stage_client = types.SimpleNamespace(
stage_type="llm",
is_comprehension=False,
final_output=True,
final_output_type=None,
default_sampling_params=types.SimpleNamespace(),
)
captured: dict[str, object] = {}
output_processor = object()
def _capture_build_llm_stage_output_processor(plan, stage_vllm_config, *, log_stats=False):
captured["plan"] = plan
captured["stage_vllm_config"] = stage_vllm_config
captured["log_stats"] = log_stats
return output_processor
monkeypatch.setattr(
runtime_mod,
"build_llm_stage_output_processor",
_capture_build_llm_stage_output_processor,
)
pools = runtime._assemble_stage_pools([stage_plan], {0: [stage_client]})
assert pools[0].output_processor is output_processor
assert captured == {
"plan": stage_plan,
"stage_vllm_config": cfg,
"log_stats": True,
}
def test_build_logical_stage_init_plans_applies_replica_device_splits(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
@@ -537,10 +661,11 @@ def test_initialize_local_llm_replica_passes_stage_init_timeout_to_complete_stag
stage_configs=[],
model="dummy-model",
config_path="dummy-config",
stage_init_timeout=302,
stage_init_timeout=1,
diffusion_batch_size=1,
async_chunk=False,
)
stage_init_timeout = 302
fake_vllm_config = types.SimpleNamespace()
fake_addresses = types.SimpleNamespace(inputs=["in"], outputs=["out"], frontend_stats_publish_address=None)
@@ -587,14 +712,14 @@ def test_initialize_local_llm_replica_passes_stage_init_timeout_to_complete_stag
)
try:
runtime._initialize_local_llm_replica(plan, 302)
runtime._initialize_local_llm_replica(plan, stage_init_timeout)
finally:
if prev_device_env is None:
os.environ.pop(device_env_var, None)
else:
os.environ[device_env_var] = prev_device_env
assert captured_timeout == 302
assert captured_timeout == stage_init_timeout
def test_build_engine_args_cli_tokenizer_overrides_inferred_base_tokenizer(tmp_path):
+168 -32
View File
@@ -15,6 +15,7 @@ import pytest
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.exceptions import EngineDeadError
from vllm.v1.metrics.stats import IterationStats
from vllm_omni.engine.messages import (
AbortRequestMessage,
@@ -145,7 +146,7 @@ class FakeStageClient:
try:
return self._engine_core_outputs.get_nowait()
except queue.Empty:
return SimpleNamespace(outputs=[])
return SimpleNamespace(outputs=[], scheduler_stats=None, finished_requests=None)
def get_diffusion_output_nowait(self):
try:
@@ -253,12 +254,40 @@ class FakeOutputProcessor:
return None
class IterationStatsOutputProcessor(FakeOutputProcessor):
def process_outputs(self, *_args, **kwargs):
iteration_stats = kwargs.get("iteration_stats")
if iteration_stats is None and len(_args) >= 3:
iteration_stats = _args[2]
assert iteration_stats is not None
iteration_stats.num_generation_tokens += 3
return super().process_outputs(*_args, **kwargs)
class RecordingOutputProcessor(FakeOutputProcessor):
def __init__(self, *, request_outputs: list[object] | None = None) -> None:
super().__init__(request_outputs=request_outputs)
self.process_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
def process_outputs(self, *args, **kwargs):
self.process_calls.append((args, kwargs))
return super().process_outputs(*args, **kwargs)
class RecordingStatLogger:
def __init__(self) -> None:
self.records: list[tuple[Any, Any, int]] = []
def record(self, scheduler_stats, iteration_stats, *, engine_idx: int = 0) -> None:
self.records.append((scheduler_stats, iteration_stats, engine_idx))
def _sampling_params(max_tokens: int = 4) -> SamplingParams:
return SamplingParams(max_tokens=max_tokens)
def _engine_core_outputs(tag: str, timestamp: float) -> SimpleNamespace:
return SimpleNamespace(outputs=[tag], timestamp=timestamp, scheduler_stats=None)
return SimpleNamespace(outputs=[tag], timestamp=timestamp, scheduler_stats=None, finished_requests=None)
def _build_request_output(
@@ -329,6 +358,7 @@ def _build_harness(
output_processors: list[object] | None = None,
stage_vllm_configs: list[object] | None = None,
async_chunk: bool = False,
log_stats: bool = False,
stage_pools: list[StagePool] | None = None,
) -> OrchestratorFixture:
"""Build an Orchestrator test harness.
@@ -364,6 +394,7 @@ def _build_harness(
rpc_async_queue=rpc_queue.async_q,
stage_pools=stage_pools,
async_chunk=async_chunk,
log_stats=log_stats,
)
ready_future.set_result((orchestrator, request_queue, output_queue, rpc_queue))
await orchestrator.run()
@@ -2137,38 +2168,143 @@ async def test_multi_replica_cfg_companion_inherits_parent_affinity(orchestrator
await _shutdown_orchestrator(orchestrator_fixture)
def test_orchestrator_does_not_re_introduce_global_stats_throttle() -> None:
"""Regression: each (stage, replica) must independently publish its wrapped
vllm:* stats when its scheduler emits non-None scheduler_stats.
A previous version of Orchestrator carried a global self._last_stats_ts /
_stats_interval_s gate around _stat_logger.record(). Because
OmniSchedulerMixin.make_stats() already throttles at 1 Hz per scheduler
(one per (stage, replica)), the extra global gate starved every replica
other than the first to emit within each second — their {stage, replica}
gauges/counters went stale.
The fix removed the global gate entirely; the only signal needed is
'this replica's scheduler emitted non-None scheduler_stats'. This test
fails loudly if someone reintroduces the global throttle.
"""
import inspect
from vllm_omni.engine.orchestrator import Orchestrator
source = inspect.getsource(Orchestrator)
assert "_last_stats_ts" not in source, (
"Orchestrator must not gate stat recording on a global timestamp. "
"OmniSchedulerMixin.make_stats() already throttles per scheduler "
"(per (stage, replica)); an outer global gate starves all but the "
"first replica to emit within each 1s window."
@pytest.mark.asyncio
async def test_orchestrator_records_iteration_stats_without_scheduler_stats(orchestrator_factory) -> None:
stage0 = FakeStageClient(stage_type="llm", final_output=True)
processor = IterationStatsOutputProcessor(
request_outputs=[_build_request_output("req-stats", token_ids=[1], finished=True)]
)
assert "_stats_interval_s" not in source
assert "raw_outputs.scheduler_stats is not None" in source, (
"Orchestrator must gate stat recording solely on "
"raw_outputs.scheduler_stats being non-None — the per-scheduler 1Hz "
"throttle in OmniSchedulerMixin.make_stats() is the only gate needed."
orchestrator_fixture = orchestrator_factory([stage0], output_processors=[processor], log_stats=True)
stat_logger = RecordingStatLogger()
orchestrator_fixture.orchestrator._stat_logger = stat_logger
request = SimpleNamespace(request_id="req-stats", prompt_token_ids=[1, 2])
try:
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-stats",
prompt=request,
original_prompt={"prompt": "hello"},
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
await _wait_for(lambda: len(stage0.add_request_calls) == 1)
stage0.push_engine_core_outputs(_engine_core_outputs("raw-without-scheduler-stats", 1.0))
await _wait_for(lambda: len(stat_logger.records) == 1)
scheduler_stats, iteration_stats, engine_idx = stat_logger.records[0]
assert scheduler_stats is None
assert iteration_stats is not None
assert iteration_stats.num_generation_tokens == 3
assert engine_idx == 0
finally:
await _shutdown_orchestrator(orchestrator_fixture)
@pytest.mark.asyncio
async def test_orchestrator_records_scheduler_stats_without_outputs(orchestrator_factory) -> None:
stage0 = FakeStageClient(stage_type="llm", final_output=True)
processor = RecordingOutputProcessor()
orchestrator_fixture = orchestrator_factory([stage0], output_processors=[processor], log_stats=True)
stat_logger = RecordingStatLogger()
orchestrator_fixture.orchestrator._stat_logger = stat_logger
scheduler_stats = SimpleNamespace(num_running_reqs=0, kv_cache_usage=0.5)
try:
stage0.push_engine_core_outputs(
SimpleNamespace(
outputs=[],
timestamp=1.0,
scheduler_stats=scheduler_stats,
finished_requests=None,
)
)
await _wait_for(lambda: len(stat_logger.records) == 1)
recorded_scheduler_stats, iteration_stats, engine_idx = stat_logger.records[0]
assert recorded_scheduler_stats is scheduler_stats
assert iteration_stats is None
assert engine_idx == 0
assert processor.process_calls[0][0][2] is None
finally:
await _shutdown_orchestrator(orchestrator_fixture)
@pytest.mark.asyncio
async def test_orchestrator_does_not_build_iteration_stats_for_finished_only_batch(orchestrator_factory) -> None:
stage0 = FakeStageClient(stage_type="llm", final_output=True)
processor = RecordingOutputProcessor()
orchestrator_fixture = orchestrator_factory([stage0], output_processors=[processor], log_stats=True)
stat_logger = RecordingStatLogger()
orchestrator_fixture.orchestrator._stat_logger = stat_logger
try:
stage0.push_engine_core_outputs(
SimpleNamespace(
outputs=[],
timestamp=1.0,
scheduler_stats=None,
finished_requests={"req-finished"},
)
)
await _wait_for(lambda: bool(processor.process_calls))
assert processor.process_calls[0][0][2] is None
assert stat_logger.records == []
finally:
await _shutdown_orchestrator(orchestrator_fixture)
@pytest.mark.asyncio
async def test_orchestrator_does_not_build_iteration_stats_without_stat_logger(orchestrator_factory) -> None:
stage0 = FakeStageClient(stage_type="llm", final_output=True)
processor = RecordingOutputProcessor()
orchestrator_fixture = orchestrator_factory([stage0], output_processors=[processor], log_stats=True)
orchestrator_fixture.orchestrator._stat_logger = None
try:
stage0.push_engine_core_outputs(_engine_core_outputs("raw-output", 1.0))
await _wait_for(lambda: bool(processor.process_calls))
assert processor.process_calls[0][0][2] is None
finally:
await _shutdown_orchestrator(orchestrator_fixture)
@pytest.mark.asyncio
async def test_stage_pool_preserves_none_iteration_stats() -> None:
client = FakeStageClient(stage_type="llm", final_output=True)
processor = RecordingOutputProcessor()
pool = StagePool(
0,
[client],
output_processor=processor,
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
)
raw_outputs = SimpleNamespace(outputs=["raw"], timestamp=1.0, scheduler_stats=None)
await pool.process_llm_raw_outputs(0, raw_outputs, iteration_stats=None)
assert processor.process_calls[0][0][2] is None
@pytest.mark.asyncio
async def test_stage_pool_process_llm_raw_outputs_mutates_iteration_stats() -> None:
client = FakeStageClient(stage_type="llm", final_output=True)
processor = IterationStatsOutputProcessor()
pool = StagePool(
0,
[client],
output_processor=processor,
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
)
raw_outputs = SimpleNamespace(outputs=["raw"], timestamp=1.0, scheduler_stats=None)
iteration_stats = IterationStats()
await pool.process_llm_raw_outputs(0, raw_outputs, iteration_stats=iteration_stats)
assert iteration_stats.num_generation_tokens == 3
@pytest.mark.asyncio
+48 -1
View File
@@ -10,9 +10,11 @@ import pytest
import torch
from vllm.outputs import PoolingRequestOutput
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine import FinishReason
from vllm.v1.engine import EngineCoreEvent, EngineCoreEventType, FinishReason
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
from vllm.v1.metrics.stats import IterationStats, PrefillStats
from vllm_omni.engine import OmniEngineCoreOutput
from vllm_omni.outputs import output_processor
from vllm_omni.outputs.output_modality import OutputModality, OutputModalityNames
from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState
@@ -616,3 +618,48 @@ def test_no_detokenizer_final_only():
result = s.make_request_output([], None, FinishReason.STOP, None)
assert result is not None
assert AUDIO in result.outputs[0].multimodal_output
def test_mm_only_outputs_update_iteration_stats():
processor = MultimodalOutputProcessor(
tokenizer=None,
log_stats=True,
engine_core_output_type=AUDIO,
)
state = _make_no_detok_state(RequestOutputKind.CUMULATIVE)
processor.request_states[state.request_id] = state
processor.external_req_ids[state.external_req_id].append(state.request_id)
prefill_stats = PrefillStats()
prefill_stats.set(
num_prompt_tokens=4,
num_local_cached_tokens=0,
num_external_cached_tokens=0,
)
output = OmniEngineCoreOutput(
request_id=state.request_id,
new_token_ids=[10, 11],
finish_reason=FinishReason.STOP,
events=[
EngineCoreEvent(EngineCoreEventType.QUEUED, 1.0),
EngineCoreEvent(EngineCoreEventType.SCHEDULED, 2.0),
],
prefill_stats=prefill_stats,
multimodal_output={AUDIO: torch.ones(1, 8)},
)
iteration_stats = IterationStats()
processor.process_outputs(
[output],
engine_core_timestamp=3.0,
iteration_stats=iteration_stats,
)
assert iteration_stats.num_prompt_tokens == 4
assert iteration_stats.num_generation_tokens == 2
assert len(iteration_stats.time_to_first_tokens_iter) == 1
assert len(iteration_stats.finished_requests) == 1
finished = iteration_stats.finished_requests[0]
assert finished.finish_reason == FinishReason.STOP
assert finished.num_prompt_tokens == state.prompt_len
assert finished.num_generation_tokens == 2
+10 -1
View File
@@ -778,6 +778,7 @@ class TestSingleStageReplicaInitialization:
single_stage_id_filter=None,
omni_master_address="127.0.0.1",
omni_master_port=26000,
log_stats=True,
)
runtime._omni_master_server = mocker.Mock(spec=OmniMasterServer)
runtime._omni_master_server.get_stage_config.return_value = {"stage_id": 1, "stage_type": "llm"}
@@ -811,10 +812,17 @@ class TestSingleStageReplicaInitialization:
sentinel_client = SimpleNamespace()
mock_connect = mocker.patch.object(runtime_mod, "connect_remote_engine_cores", side_effect=_fake_connect)
client_kwargs: dict[str, Any] = {}
def _capture_make_async_mp_client(**kwargs):
client_kwargs.update(kwargs)
events.append("attach")
return sentinel_client
mocker.patch.object(
StageEngineCoreClientBase,
"make_async_mp_client",
side_effect=lambda **_: (events.append("attach"), sentinel_client)[1],
side_effect=_capture_make_async_mp_client,
)
result = runtime._initialize_remote_replica(plan, stage_init_timeout=60)
@@ -824,6 +832,7 @@ class TestSingleStageReplicaInitialization:
assert mock_connect.call_args.kwargs["vllm_config"].parallel_config.data_parallel_size_local == 0
assert mock_connect.call_args.kwargs["stage_id"] == 1
assert mock_connect.call_args.kwargs["replica_id"] == 0
assert client_kwargs["log_stats"] is True
assert events == ["enter", "exit", "attach"]
def test_initialize_llm_replica_remote_missing_registered_stage_config_raises(self, mocker: MockerFixture):
@@ -1301,7 +1301,11 @@ def test_generate_images_rejects_model_mismatch(test_client):
assert "model mismatch" in response.json()["detail"].lower()
def test_image_file_response_format_multiple(test_client):
@pytest.mark.parametrize(
("output_format", "expected_pil_format"),
[("png", "PNG"), ("jpeg", "JPEG"), ("webp", "WEBP")],
)
def test_image_file_response_format_multiple(test_client, output_format, expected_pil_format):
"""Test response_format=file with n>1 returns ZIP archive"""
response = test_client.post(
"/v1/images/generations",
@@ -1309,6 +1313,7 @@ def test_image_file_response_format_multiple(test_client):
"prompt": "a dog",
"n": 3,
"response_format": "file",
"output_format": output_format,
},
)
@@ -1317,23 +1322,32 @@ def test_image_file_response_format_multiple(test_client):
assert "attachment" in response.headers.get("content-disposition", "")
assert ".zip" in response.headers.get("content-disposition", "")
# Verify it's a valid ZIP with 3 PNG files
# Verify it's a valid ZIP with 3 correctly labelled image files
import zipfile
zip_buffer = io.BytesIO(response.content)
with zipfile.ZipFile(zip_buffer, "r") as zf:
files = zf.namelist()
assert len(files) == 3
assert all(f.endswith(".png") for f in files)
assert all(f.endswith(f".{output_format}") for f in files)
# Verify each file is a valid PNG
# Verify each file's bytes match its advertised format
for filename in files:
img_bytes = zf.read(filename)
img = Image.open(io.BytesIO(img_bytes))
assert img.format == "PNG"
assert img.format == expected_pil_format
def test_image_file_response_format_single(test_client):
@pytest.mark.parametrize(
("output_format", "expected_media_type", "expected_pil_format"),
[
("png", "image/png", "PNG"),
("jpg", "image/jpeg", "JPEG"),
("jpeg", "image/jpeg", "JPEG"),
("webp", "image/webp", "WEBP"),
],
)
def test_image_file_response_format_single(test_client, output_format, expected_media_type, expected_pil_format):
"""Test response_format=file with n=1 returns a single image file."""
response = test_client.post(
"/v1/images/generations",
@@ -1341,16 +1355,61 @@ def test_image_file_response_format_single(test_client):
"prompt": "a dog",
"n": 1,
"response_format": "file",
"output_format": output_format,
},
)
assert response.status_code == 200
assert response.headers["content-type"] == "image/png"
assert response.headers["content-type"] == expected_media_type
assert "attachment" in response.headers.get("content-disposition", "")
assert ".png" in response.headers.get("content-disposition", "")
assert f".{output_format}" in response.headers.get("content-disposition", "")
img = Image.open(io.BytesIO(response.content))
assert img.format == "PNG"
assert img.format == expected_pil_format
@pytest.mark.parametrize(
("output_format", "image_count", "expected_media_type", "expected_pil_format"),
[
("jpeg", 1, "image/jpeg", "JPEG"),
("webp", 2, "application/zip", "WEBP"),
],
)
def test_multistage_image_file_response_uses_requested_format(
async_omni_test_client,
output_format,
image_count,
expected_media_type,
expected_pil_format,
):
engine = async_omni_test_client.app.state.engine_client
engine._images = [Image.new("RGB", (16, 16), color="green") for _ in range(image_count)]
response = async_omni_test_client.post(
"/v1/images/generations",
json={
"prompt": "a dog",
"n": image_count,
"response_format": "file",
"output_format": output_format,
},
)
assert response.status_code == 200
assert response.headers["content-type"] == expected_media_type
if image_count == 1:
assert f".{output_format}" in response.headers["content-disposition"]
assert Image.open(io.BytesIO(response.content)).format == expected_pil_format
return
import zipfile
with zipfile.ZipFile(io.BytesIO(response.content), "r") as archive:
filenames = archive.namelist()
assert len(filenames) == image_count
assert all(filename.endswith(f".{output_format}") for filename in filenames)
for filename in filenames:
assert Image.open(io.BytesIO(archive.read(filename))).format == expected_pil_format
def make_test_image_bytes(size=(64, 64)) -> bytes:
@@ -48,6 +48,25 @@ def test_encode_video_bytes_exports_frames_without_interpolation(monkeypatch):
assert mux_calls[0]["audio"] is None
@pytest.mark.parametrize("frame_count", [3, 4])
def test_channel_last_video_tensor_preserves_channel_sized_frame_count(frame_count):
video = torch.arange(frame_count * 2 * 5 * 3, dtype=torch.uint8).reshape(frame_count, 2, 5, 3)
frames = video_api_utils._coerce_video_to_uint8_frames(video)
assert frames.shape == (frame_count, 2, 5, 3)
np.testing.assert_array_equal(frames, video.numpy())
def test_channel_first_video_tensor_is_converted_to_channel_last():
video = torch.arange(3 * 5 * 2 * 6, dtype=torch.uint8).reshape(3, 5, 2, 6)
frames = video_api_utils._coerce_video_to_uint8_frames(video)
assert frames.shape == (5, 2, 6, 3)
np.testing.assert_array_equal(frames, video.permute(1, 2, 3, 0).numpy())
def test_fragmented_mp4_video_encoder_reuses_single_muxer(monkeypatch):
muxers = []
+89 -11
View File
@@ -17,6 +17,7 @@ Key invariants tested:
"""
import json
import time
from unittest.mock import MagicMock
import pytest
@@ -80,28 +81,37 @@ def _make_audio_omni_output(
request_id: str = "test-req",
index: int = 0,
num_prompt_tokens: int = 3,
stage_id: int | None = None,
replica_id: int | None = None,
audio_samples: int = 0,
) -> OmniRequestOutput:
"""Build an OmniRequestOutput for audio (no torch dependency)."""
completion = CompletionOutput(
index=index,
text="",
token_ids=[],
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
)
if audio_samples > 0:
completion.multimodal_output = {
"audio": [MagicMock(numel=MagicMock(return_value=audio_samples))],
"sr": 24000,
}
res = RequestOutput(
request_id=request_id,
prompt="test",
prompt_token_ids=list(range(num_prompt_tokens)),
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=index,
text="",
token_ids=[],
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
)
],
outputs=[completion],
finished=True,
)
return OmniRequestOutput(
request_id=request_id,
stage_id=stage_id,
replica_id=replica_id,
final_output_type="audio",
request_output=res,
finished=True,
@@ -321,6 +331,74 @@ async def test_single_modality_audio_only_one_stop():
assert finish_reasons[-1] == "stop"
@pytest.mark.parametrize(
("output_replica_id", "pool_replica_id", "expected_replica_id", "pool_lookup"),
[
(7, None, 7, False),
(None, 5, 5, True),
],
)
@pytest.mark.asyncio
async def test_streaming_audio_metrics_resolve_replica_id(
monkeypatch,
output_replica_id,
pool_replica_id,
expected_replica_id,
pool_lookup,
):
import vllm_omni.entrypoints.openai.serving_chat as serving_chat_mod
from vllm_omni.entrypoints.client_request_state import ClientRequestState
serving_chat = _build_serving_chat()
request = _make_request(modalities=["audio"])
req_state = ClientRequestState(
request_id="internal-req",
external_request_id="test-req",
)
req_state.request_arrival_ts = time.time() - 1.0
serving_chat.engine_client.request_states = {"internal-req": req_state}
serving_chat.engine_client.mod_metrics = object()
serving_chat.engine_client.engine = MagicMock()
serving_chat.engine_client.engine.stage_pools = [MagicMock(), MagicMock(), MagicMock()]
serving_chat.engine_client.engine.stage_pools[2].get_bound_replica_id.return_value = pool_replica_id
first_packet_calls = []
finalize_calls = []
def fake_observe_audio_first_packet(*args, **kwargs):
first_packet_calls.append(kwargs)
def fake_observe_audio_streaming_finalize(*args, **kwargs):
finalize_calls.append(kwargs)
monkeypatch.setattr(serving_chat_mod, "observe_audio_first_packet", fake_observe_audio_first_packet)
monkeypatch.setattr(serving_chat_mod, "observe_audio_streaming_finalize", fake_observe_audio_streaming_finalize)
async def result_generator():
yield _make_audio_omni_output(stage_id=2, replica_id=output_replica_id, audio_samples=2400)
await _collect_stream(
serving_chat.chat_completion_stream_generator(
request=request,
result_generator=result_generator(),
request_id="test-req",
model_name="test-model",
conversation=[],
tokenizer=MagicMock(),
request_metadata=MagicMock(),
)
)
if pool_lookup:
serving_chat.engine_client.engine.stage_pools[2].get_bound_replica_id.assert_called_once_with("internal-req")
else:
serving_chat.engine_client.engine.stage_pools[2].get_bound_replica_id.assert_not_called()
assert first_packet_calls[0]["stage_id"] == 2
assert first_packet_calls[0]["replica_id"] == expected_replica_id
assert finalize_calls[0]["stage_id"] == 2
assert finalize_calls[0]["replica_id"] == expected_replica_id
# ---------------------------------------------------------------------------
# Tests: fallback stop chunk when declared modality is not produced
# ---------------------------------------------------------------------------
+6 -3
View File
@@ -527,7 +527,7 @@ def _resolve_audio_transcript(
audio_bytes = getattr(response, "audio_bytes", None)
if not audio_bytes:
return None
return convert_audio_bytes_to_text(audio_bytes)
return convert_audio_bytes_to_text(audio_bytes, language=request_config.get("transcript_language"))
def assert_omni_response(response: Any, request_config: dict[str, Any], run_level):
@@ -632,6 +632,7 @@ def _assert_transcript_matches(
*,
threshold: float,
escalation_model: str | None = None,
language: str | None = None,
) -> None:
"""Assert spoken audio matches ``expected_text``.
@@ -647,7 +648,8 @@ def _assert_transcript_matches(
clip is re-transcribed with that stronger ASR and the assertion is decided on
its verdict -- so a weak whisper-``small`` mishear on a short clip does not
flake the gate, while a genuine model artifact still fails (the strong ASR
mismatches too).
mismatches too). ``language`` applies to that escalated pass too, so a request
that pins a language does not silently fall back to auto-detection on retry.
"""
expected = str(expected_text).strip().lower()
similarity = cosine_similarity_text(transcript.strip().lower(), expected)
@@ -664,7 +666,7 @@ def _assert_transcript_matches(
f"whisper-small below threshold ({similarity:.2f} <= {threshold}); "
f"escalating to whisper-{escalation_model} to rule out an ASR mishear"
)
strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model)
strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model, language=language)
strong_similarity = cosine_similarity_text(strong_transcript.strip().lower(), expected)
print(
f"audio content (whisper-{escalation_model}): {strong_transcript}\n"
@@ -741,6 +743,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
expected_text,
threshold=0.9,
escalation_model=request_config.get("transcript_escalation_model"),
language=request_config.get("transcript_language"),
)
_assert_preset_voice_gender_from_audio(
response.audio_bytes,
+10 -5
View File
@@ -652,7 +652,9 @@ def _serialize_whisper_model_download(model_size: str = "small"):
f.close()
def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "small") -> str:
def _whisper_transcribe_in_current_process(
output_path: str, model_size: str = "small", language: str | None = None
) -> str:
import whisper
device_index = None
@@ -684,6 +686,9 @@ def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "
temperature=0.0,
word_timestamps=True,
condition_on_previous_text=False,
# None keeps whisper's auto-detection. Do not default this to a
# language: callers include non-English audio tests.
language=language,
)["text"]
finally:
del model
@@ -694,15 +699,15 @@ def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "
return text or ""
def convert_audio_file_to_text(output_path: str, model_size: str = "small") -> str:
def convert_audio_file_to_text(output_path: str, model_size: str = "small", language: str | None = None) -> str:
"""Convert an audio file to text in an isolated subprocess."""
ctx = multiprocessing.get_context("spawn")
with concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=ctx) as executor:
future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size)
future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size, language)
return future.result()
def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") -> str:
def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small", language: str | None = None) -> str:
output_fd, output_path = tempfile.mkstemp(prefix="test_", suffix=".wav")
os.close(output_fd)
if os.environ.get("VLLM_OMNI_KEEP_REQUEST_MEDIA", "").lower() not in ("1", "true", "yes"):
@@ -710,7 +715,7 @@ def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") ->
data, samplerate = sf.read(io.BytesIO(raw_bytes))
sf.write(output_path, data, samplerate, format="WAV", subtype="PCM_16")
print(f"audio data is saved: {output_path}")
return convert_audio_file_to_text(output_path, model_size)
return convert_audio_file_to_text(output_path, model_size, language)
__all__ = [
+13
View File
@@ -24,6 +24,19 @@ DEPLOY_CONFIG = modify_stage_config(
1: {"kv_cache_memory_bytes": 512 * 1024 * 1024},
2: {"kv_cache_memory_bytes": 256 * 1024 * 1024},
},
# Platform overrides are applied after ordinary stage settings. Keep
# the constrained duplex test's Talker budget ahead of the base CUDA
# profile's single-GPU default.
"platforms": {
"cuda": {
"stages": [
{
"stage_id": 1,
"kv_cache_memory_bytes": 512 * 1024 * 1024,
}
]
}
},
},
)
CORE_DEPLOY_CONFIG = modify_stage_config(
+64 -1
View File
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
from tests.helpers.assertions import _assert_transcript_matches
from tests.helpers import assertions
from tests.helpers.assertions import _assert_transcript_matches, _resolve_audio_transcript
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -25,3 +28,63 @@ def test_short_transcript_unrelated_text_still_fails():
expected_text="how are you",
threshold=0.9,
)
def _capture_transcribe(monkeypatch) -> dict:
captured: dict = {}
def fake_convert(raw_bytes, model_size="small", language=None):
captured["model_size"] = model_size
captured["language"] = language
return "London"
monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", fake_convert)
return captured
def test_resolve_transcript_honors_declared_language(monkeypatch):
captured = _capture_transcribe(monkeypatch)
response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
_resolve_audio_transcript(
response,
{"modalities": ["text", "audio"], "transcript_language": "en"},
"full_model",
speech_api=False,
)
assert captured["language"] == "en"
def test_resolve_transcript_leaves_language_unset_by_default(monkeypatch):
captured = _capture_transcribe(monkeypatch)
response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
_resolve_audio_transcript(
response,
{"modalities": ["text", "audio"]},
"full_model",
speech_api=False,
)
assert captured["language"] is None
assert captured["model_size"] == "small"
def test_escalated_transcript_keeps_declared_language(monkeypatch):
# The escalated pass must honour the same language, otherwise a request that
# pins one silently falls back to auto-detection on retry.
captured = _capture_transcribe(monkeypatch)
with pytest.raises(AssertionError, match="after ASR escalation"):
_assert_transcript_matches(
"totally different words",
audio_bytes=b"fake-wav",
expected_text="how are you",
threshold=0.9,
escalation_model="large-v3",
language="en",
)
assert captured["model_size"] == "large-v3"
assert captured["language"] == "en"
+81
View File
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from types import SimpleNamespace
import numpy as np
import pytest
import soundfile as sf
from tests.helpers import media
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _stub_transcribe(monkeypatch) -> dict:
"""Capture the kwargs the helper hands to whisper, with no model and no device."""
captured: dict = {}
class FakeModel:
def transcribe(self, path, **kwargs):
captured.update(kwargs)
return {"text": "London"}
monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace(load_model=lambda size, device=None: FakeModel()))
# Keep the unit test off the accelerator probe so it stays hermetic.
monkeypatch.setitem(
sys.modules,
"vllm_omni.platforms",
SimpleNamespace(current_omni_platform=SimpleNamespace(is_available=lambda: False)),
)
return captured
def test_transcribe_forwards_requested_language(monkeypatch):
captured = _stub_transcribe(monkeypatch)
media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small", language="en")
assert captured["language"] == "en"
def test_transcribe_defaults_to_auto_language(monkeypatch):
# Auto-detect must stay the default: forcing a language globally would break
# the non-English audio tests (e.g. the Chinese Qwen3-Omni prompts).
captured = _stub_transcribe(monkeypatch)
media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small")
assert captured.get("language") is None
def test_bytes_entrypoint_forwards_language_to_subprocess(monkeypatch, tmp_path):
"""Cover the two hops the tests above skip: bytes -> file -> executor.submit.
The real call crosses a spawn ProcessPoolExecutor, so capture what gets
submitted rather than what the worker eventually does.
"""
submitted: dict = {}
class FakeExecutor:
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def submit(self, fn, *args):
submitted["fn"] = fn
submitted["args"] = args
return SimpleNamespace(result=lambda: "London")
monkeypatch.setattr(media.concurrent.futures, "ProcessPoolExecutor", lambda *a, **kw: FakeExecutor())
wav = tmp_path / "clip.wav"
sf.write(wav, np.zeros(2400, dtype=np.float32), 24000)
assert media.convert_audio_bytes_to_text(wav.read_bytes(), "small", "en") == "London"
assert submitted["fn"] is media._whisper_transcribe_in_current_process
assert submitted["args"][1:] == ("small", "en")
@@ -170,7 +170,12 @@ class TestDeployTopology:
0.15,
]
assert sum(memory_utilizations) <= 0.9 + 1e-6
assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}}
# Daily-Omni minicpm-interleave: up to 64 image/audio items (+ optional video).
assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {
"image": 64,
"audio": 64,
"video": 1,
}
elif filename in {"minicpmo_4_5_2gpu.yaml"}:
assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [
0.9,
+35
View File
@@ -17,6 +17,9 @@ from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm_omni.distributed.omni_connectors.kv_transfer_manager import (
OmniKVTransferManager,
)
from vllm_omni.outputs import OmniConnectorOutput
from vllm_omni.worker.omni_connector_model_runner_mixin import (
OmniConnectorModelRunnerMixin,
@@ -1198,6 +1201,38 @@ class TestRankAwareKVRouting(unittest.TestCase):
)
host.shutdown_omni_connectors()
def test_single_rank_hooks_use_default_manager_key(self):
host = self._make_host(from_tp=1, to_tp=1, local_rank=0)
expected = ["omni_0_to_1_kv_cache_req"]
self.assertEqual(host.get_rank_aware_kv_send_keys("req", from_stage=0, to_stage=1), expected)
self.assertEqual(host.get_rank_aware_kv_keys("req", from_stage=0, to_stage=1), expected)
host.shutdown_omni_connectors()
def test_init_uses_effective_manager_tp2_topology(self):
model_config = _make_model_config(stage_id=0)
model_config.omni_kv_config = {
"need_send_cache": True,
"omni_from_stage": "0",
"omni_to_stage": "1",
"rank_mapping": {"from_tp": 2, "to_tp": 2},
}
with patch(
"vllm_omni.distributed.omni_connectors.kv_transfer_manager.get_local_tp_rank",
return_value=1,
):
manager = OmniKVTransferManager.from_model_config(model_config)
host = MixinHost()
host.init_omni_connectors(model_config=model_config, kv_transfer_manager=manager)
expected = ["req_0_0_1_1"]
self.assertEqual(host.get_rank_aware_kv_send_keys("req", from_stage=0, to_stage=1), expected)
self.assertEqual(host.get_rank_aware_kv_keys("req", from_stage=0, to_stage=1), expected)
self.assertEqual(host.get_kv_rank_mapping()["from_tp"], 2)
self.assertEqual(host.get_kv_rank_mapping()["to_tp"], 2)
host.shutdown_omni_connectors()
def test_send_keys_route_from_rank_gt_to_rank(self):
host = self._make_host(from_tp=4, to_tp=2, local_rank=3)
self.assertEqual(host.get_rank_aware_kv_send_keys("req", from_stage=0), ["req_0_0_3_1"])