Merge branch 'main' into fix/tp-local-kv-recv-consensus
This commit is contained in:
61 files changed
+1881
-224
No files matched your search
@@ -0,0 +1,249 @@
|
||||
"""Tests for Daily-Omni ``--daily-omni-pack-mode`` packing recipes.
|
||||
|
||||
The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
|
||||
OpenBMB's reported Daily-Omni score when frames and 1s audio segments arrive strictly
|
||||
interleaved, with ``max_slice_nums=1`` / ``use_image_id=False`` and an empty MiniCPM
|
||||
system message (not the Qwen system prompt). These tests pin that contract so a refactor
|
||||
cannot silently regress the score.
|
||||
|
||||
vllm stubs are installed by tests/benchmarks/conftest.py before collection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "daily_omni_dataset.py"
|
||||
_MODULE_NAME = "vllm_omni.benchmarks.data_modules.daily_omni_dataset"
|
||||
|
||||
if _MODULE_NAME not in sys.modules:
|
||||
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_MODULE_NAME] = _mod
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( # noqa: E402
|
||||
DailyOmniDataset,
|
||||
_uniform_sample_indices,
|
||||
)
|
||||
|
||||
_VIDEO_ID = "vid001"
|
||||
_NUM_FRAMES = 5
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_dir(tmp_path: Path) -> Path:
|
||||
"""Official Daily-Omni ``Videos/`` layout with placeholder media files."""
|
||||
per_video = tmp_path / "Videos" / _VIDEO_ID
|
||||
per_video.mkdir(parents=True)
|
||||
(per_video / f"{_VIDEO_ID}_video.mp4").write_bytes(b"fake-mp4")
|
||||
(per_video / f"{_VIDEO_ID}_audio.wav").write_bytes(b"fake-wav")
|
||||
return tmp_path / "Videos"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def qa_json(tmp_path: Path) -> Path:
|
||||
path = tmp_path / "qa.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"video_id": _VIDEO_ID,
|
||||
"question": "What happens first?",
|
||||
"choice": {"A": "a", "B": "b", "C": "c", "D": "d"},
|
||||
"answer": "A",
|
||||
"task_type": "AV Event Alignment",
|
||||
"video_duration": "30s",
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _make_dataset(qa_json: Path, video_dir: Path, **kwargs) -> DailyOmniDataset:
|
||||
return DailyOmniDataset(
|
||||
qa_json_path=str(qa_json),
|
||||
video_dir=str(video_dir),
|
||||
disable_shuffle=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) -> None:
|
||||
"""Replace decord/ffmpeg extraction with deterministic in-memory frames + segments."""
|
||||
from PIL import Image
|
||||
|
||||
def fake_extract(video_path, *, audio_path, include_audio, **_kw):
|
||||
counter.append(1)
|
||||
frames = [Image.new("RGB", (16, 16), color=(i, i, i)) for i in range(_NUM_FRAMES)]
|
||||
segments = [np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
|
||||
return frames, segments
|
||||
|
||||
monkeypatch.setattr(dataset, "_extract_minicpm_frame_audio_segments", fake_extract)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _uniform_sample_indices
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "k"), [(10, 4), (100, 64), (1000, 64), (7, 7), (3, 5), (1, 1)])
|
||||
def test_uniform_sample_indices_matches_openbmb_midpoint(n: int, k: int) -> None:
|
||||
"""Pin OpenBMB midpoint-bin sampling (not np.linspace endpoints)."""
|
||||
got = _uniform_sample_indices(n, k)
|
||||
if n <= k:
|
||||
assert got == list(range(n))
|
||||
else:
|
||||
gap = n / k
|
||||
expected = [int(i * gap + gap / 2) for i in range(k)]
|
||||
assert got == expected
|
||||
# Endpoints differ from linspace for typical Daily-Omni downsample sizes.
|
||||
linspace = [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
|
||||
if n == 1000 and k == 64:
|
||||
assert got[0] == 7 and got[-1] == 992
|
||||
assert linspace[0] == 0 and linspace[-1] == 999
|
||||
assert got != linspace
|
||||
|
||||
|
||||
def test_uniform_sample_indices_non_positive_k() -> None:
|
||||
assert _uniform_sample_indices(10, 0) == []
|
||||
assert _uniform_sample_indices(10, -1) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# minicpm-interleave packing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_minicpm_interleave_alternates_image_and_audio(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, extra, position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert position == "first"
|
||||
assert [p["type"] for p in parts] == ["image_url", "audio_url"] * _NUM_FRAMES
|
||||
assert extra["mm_processor_kwargs"] == {
|
||||
"use_audio_in_video": False,
|
||||
"max_slice_nums": 1,
|
||||
"use_image_id": False,
|
||||
}
|
||||
|
||||
|
||||
def test_minicpm_interleave_visual_mode_emits_frames_only(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="visual")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert [p["type"] for p in parts] == ["image_url"] * _NUM_FRAMES
|
||||
|
||||
|
||||
def test_minicpm_interleave_writes_file_urls_under_video_dir(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
segment_dir = video_dir / ".minicpm_daily_omni_interleave" / _VIDEO_ID
|
||||
for part in parts:
|
||||
url = part[part["type"]]["url"]
|
||||
assert url.startswith("file://")
|
||||
assert Path(url[len("file://") :]).parent == segment_dir.resolve()
|
||||
assert sorted(p.name for p in segment_dir.glob("frame_*.jpg")) == [f"frame_{i:04d}.jpg" for i in range(_NUM_FRAMES)]
|
||||
|
||||
|
||||
def test_minicpm_interleave_inline_mode_emits_data_urls(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(
|
||||
qa_json,
|
||||
video_dir,
|
||||
pack_mode="minicpm-interleave",
|
||||
input_mode="all",
|
||||
inline_local_video=True,
|
||||
)
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert parts[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
assert parts[1]["audio_url"]["url"].startswith("data:audio/wav;base64,")
|
||||
assert not (video_dir / ".minicpm_daily_omni_interleave").exists()
|
||||
|
||||
|
||||
def test_minicpm_interleave_caches_file_parts_but_not_inline_payloads(monkeypatch, qa_json, video_dir) -> None:
|
||||
"""Inline base64 must not be retained: ~700 Daily-Omni videos would pin GBs of RSS."""
|
||||
file_calls: list[int] = []
|
||||
ds_file = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds_file, file_calls)
|
||||
ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
assert len(file_calls) == 1
|
||||
assert ds_file._minicpm_interleave_cache
|
||||
|
||||
inline_calls: list[int] = []
|
||||
ds_inline = _make_dataset(
|
||||
qa_json,
|
||||
video_dir,
|
||||
pack_mode="minicpm-interleave",
|
||||
input_mode="all",
|
||||
inline_local_video=True,
|
||||
)
|
||||
_stub_extract(monkeypatch, ds_inline, inline_calls)
|
||||
ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
assert len(inline_calls) == 2
|
||||
assert not ds_inline._minicpm_interleave_cache
|
||||
|
||||
|
||||
def test_minicpm_interleave_missing_local_video_returns_none(qa_json, tmp_path) -> None:
|
||||
empty = tmp_path / "empty_videos"
|
||||
empty.mkdir()
|
||||
ds = _make_dataset(qa_json, empty, pack_mode="minicpm-interleave", input_mode="all")
|
||||
|
||||
parts, extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert parts is None
|
||||
assert extra is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System prompt selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_minicpm_interleave_keeps_empty_system_message(monkeypatch, qa_json, video_dir) -> None:
|
||||
"""Official MiniCPM omni mode still sends role=system with empty content."""
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
messages = ds._build_daily_omni_openai_messages(parts, "q?", {"A": "a"})
|
||||
|
||||
assert [m["role"] for m in messages] == ["system", "user"]
|
||||
assert messages[0]["content"][0]["text"] == ""
|
||||
|
||||
|
||||
def test_qwen_pack_mode_keeps_system_message(qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="qwen", input_mode="all")
|
||||
|
||||
messages = ds._build_daily_omni_openai_messages([], "q?", {"A": "a"})
|
||||
|
||||
assert [m["role"] for m in messages] == ["system", "user"]
|
||||
assert "Qwen" in messages[0]["content"][0]["text"]
|
||||
|
||||
|
||||
def test_invalid_pack_mode_rejected(qa_json, video_dir) -> None:
|
||||
with pytest.raises(ValueError, match="pack_mode must be"):
|
||||
_make_dataset(qa_json, video_dir, pack_mode="bogus")
|
||||
@@ -2214,6 +2214,32 @@ class TestPlatformOverrides:
|
||||
assert rocm.stages[0].enforce_eager is None
|
||||
assert rocm.stages[1].enforce_eager is True
|
||||
|
||||
def test_minicpmo_4_5_cuda_caps_talker_kv_cache(self):
|
||||
pipeline = resolve_pipeline_config("minicpmo_4_5")
|
||||
assert isinstance(pipeline, PipelineConfig)
|
||||
deploy_path = Path(get_deploy_config_path("minicpmo_4_5.yaml"))
|
||||
|
||||
cuda = _apply_platform_overrides(load_deploy_config(deploy_path), platform="cuda")
|
||||
cuda_stages = merge_pipeline_deploy(pipeline, cuda)
|
||||
assert cuda_stages[1].yaml_engine_args["kv_cache_memory_bytes"] == 2 * 1024**3
|
||||
|
||||
# The CUDA memory budget must not leak into the existing NPU profile.
|
||||
npu = _apply_platform_overrides(load_deploy_config(deploy_path), platform="npu")
|
||||
npu_stages = merge_pipeline_deploy(pipeline, npu)
|
||||
assert "kv_cache_memory_bytes" not in npu_stages[1].yaml_engine_args
|
||||
|
||||
# Multi-GPU replica profiles place Talker on dedicated devices and
|
||||
# preserve their previous automatic KV-cache sizing.
|
||||
for filename in (
|
||||
"minicpmo_4_5_3gpu_stage1_replicas.yaml",
|
||||
"minicpmo_4_5_4gpu_stage1_replicas.yaml",
|
||||
):
|
||||
replica = _apply_platform_overrides(
|
||||
load_deploy_config(Path(get_deploy_config_path(filename))), platform="cuda"
|
||||
)
|
||||
replica_stages = merge_pipeline_deploy(pipeline, replica)
|
||||
assert replica_stages[1].yaml_engine_args["kv_cache_memory_bytes"] is None
|
||||
|
||||
def test_npu_overrides(self):
|
||||
deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml"))
|
||||
if not deploy_path.exists():
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from vllm import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.engine.async_engine_utils import apply_omni_final_stage_metadata
|
||||
from vllm_omni.engine.serialization import deserialize_additional_information
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_engine_request() -> EngineCoreRequest:
|
||||
return EngineCoreRequest(
|
||||
request_id="req",
|
||||
prompt_token_ids=[1],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
)
|
||||
|
||||
|
||||
def _request_omits_kv_transfer(*, force_kv_transfer: bool) -> tuple[bool, dict]:
|
||||
tagged = apply_omni_final_stage_metadata(
|
||||
_make_engine_request(),
|
||||
final_stage_id=0,
|
||||
force_kv_transfer=force_kv_transfer,
|
||||
)
|
||||
scheduler = OmniARScheduler.__new__(OmniARScheduler)
|
||||
scheduler._omits_kv_transfer_cache = {}
|
||||
request = SimpleNamespace(
|
||||
request_id="req",
|
||||
additional_information=tagged.additional_information,
|
||||
)
|
||||
result = scheduler._request_omits_kv_transfer_to_next_stage(request)
|
||||
metadata = deserialize_additional_information(tagged.additional_information)
|
||||
return result, metadata
|
||||
|
||||
|
||||
def test_stage_zero_request_omits_kv_transfer():
|
||||
omits_transfer, metadata = _request_omits_kv_transfer(force_kv_transfer=False)
|
||||
|
||||
assert omits_transfer
|
||||
assert "omni_force_kv_transfer" not in metadata
|
||||
|
||||
|
||||
def test_cfg_companion_forces_kv_transfer_without_downstream_payload():
|
||||
omits_transfer, metadata = _request_omits_kv_transfer(force_kv_transfer=True)
|
||||
|
||||
assert not omits_transfer
|
||||
assert metadata["omni_final_stage_id"] == 0
|
||||
assert metadata["omni_force_kv_transfer"] is True
|
||||
@@ -21,6 +21,7 @@ import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.engine import EngineCoreEventType
|
||||
from vllm.v1.metrics.stats import PrefillStats
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
@@ -93,6 +94,17 @@ def _make_update(**kwargs) -> StreamingUpdate:
|
||||
return StreamingUpdate(**defaults)
|
||||
|
||||
|
||||
def test_generation_scheduler_records_prefill_stats_for_metrics() -> None:
|
||||
request = _make_request(prompt_token_ids=[1, 2, 3, 4])
|
||||
request.prefill_stats = PrefillStats()
|
||||
|
||||
OmniGenerationScheduler._record_prefill_stats(request)
|
||||
|
||||
assert request.prefill_stats.num_prompt_tokens == 4
|
||||
assert request.prefill_stats.num_computed_tokens == 4
|
||||
assert request.prefill_stats.num_cached_tokens == 0
|
||||
|
||||
|
||||
def test_resumable_generation_stop_marks_segment_boundary() -> None:
|
||||
session = _make_request(request_id="req-generation-segment")
|
||||
session.status = RequestStatus.RUNNING
|
||||
|
||||
@@ -9,7 +9,9 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import Wan22Pipeline
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
@@ -91,7 +93,20 @@ def _make_pipeline() -> Wan22Pipeline:
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_forward_delegates_denoising_to_diffuse() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("sampling_params_kwargs", "expected_low", "expected_high"),
|
||||
[
|
||||
({}, 4.0, 4.0),
|
||||
({"guidance_scale": 0.0}, 0.0, 0.0),
|
||||
({"guidance_scale": 1.0}, 1.0, 1.0),
|
||||
({"guidance_scale": 3.0, "guidance_scale_2": 5.0}, 3.0, 5.0),
|
||||
],
|
||||
)
|
||||
def test_forward_delegates_denoising_to_diffuse(
|
||||
sampling_params_kwargs: dict[str, float],
|
||||
expected_low: float,
|
||||
expected_high: float,
|
||||
) -> None:
|
||||
pipeline = _make_pipeline()
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
@@ -101,25 +116,15 @@ def test_forward_delegates_denoising_to_diffuse() -> None:
|
||||
|
||||
pipeline.diffuse = _fake_diffuse # type: ignore[method-assign]
|
||||
|
||||
mock_req = SimpleNamespace(
|
||||
mock_req = OmniDiffusionRequest(
|
||||
prompt="prompt",
|
||||
request_id="test-req",
|
||||
sampling_params=SimpleNamespace(
|
||||
height=None,
|
||||
width=None,
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
num_frames=1,
|
||||
num_inference_steps=2,
|
||||
guidance_scale_provided=True,
|
||||
guidance_scale=1.0,
|
||||
guidance_scale_2=None,
|
||||
boundary_ratio=None,
|
||||
generator=None,
|
||||
seed=None,
|
||||
num_outputs_per_prompt=1,
|
||||
max_sequence_length=32,
|
||||
latents=None,
|
||||
output_type="latent",
|
||||
extra_args={},
|
||||
**sampling_params_kwargs,
|
||||
),
|
||||
)
|
||||
batch = DiffusionRequestBatch(requests=[mock_req])
|
||||
@@ -129,8 +134,8 @@ def test_forward_delegates_denoising_to_diffuse() -> None:
|
||||
assert torch.equal(output.output, torch.ones((1, 4, 1, 8, 8)))
|
||||
assert torch.equal(captured["prompt_embeds"], torch.zeros(1, 32, 8))
|
||||
assert torch.equal(captured["timesteps"], pipeline.scheduler.timesteps)
|
||||
assert captured["guidance_low"] == 1.0
|
||||
assert captured["guidance_high"] == 1.0
|
||||
assert captured["guidance_low"] == expected_low
|
||||
assert captured["guidance_high"] == expected_high
|
||||
assert captured["boundary_timestep"] == pytest.approx(875.0)
|
||||
assert captured["latent_condition"] is None
|
||||
assert captured["first_frame_mask"] is None
|
||||
|
||||
@@ -72,3 +72,40 @@ def test_tp_seed_same_across_ranks_and_varies_across_requests():
|
||||
|
||||
# Seeds must vary across requests (non-determinism preserved).
|
||||
assert len(set(seeds)) == n_requests, f"Expected {n_requests} unique seeds but got {len(set(seeds))}: {seeds}"
|
||||
|
||||
|
||||
def _request_with_guidance(**sampling_kwargs) -> OmniDiffusionRequest:
|
||||
return OmniDiffusionRequest(
|
||||
prompt={"prompt": "a cup of coffee on a table"},
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1, **sampling_kwargs),
|
||||
request_id="request-test",
|
||||
)
|
||||
|
||||
|
||||
def test_omitted_guidance_scale_is_unset_and_resolved_to_one():
|
||||
# No guidance_scale passed: the caller did not set it, so it must NOT count as
|
||||
# "provided" (each pipeline applies its own default), and __post_init__ resolves the
|
||||
# None "unset" sentinel to a concrete float so no consumer ever observes None.
|
||||
req = _request_with_guidance()
|
||||
|
||||
assert req.sampling_params.guidance_scale_provided is False
|
||||
assert req.sampling_params.guidance_scale == 1.0
|
||||
|
||||
|
||||
def test_explicit_zero_guidance_scale_is_honored():
|
||||
# Regression (#4998): an explicit guidance_scale=0 (disable CFG) must be honored,
|
||||
# not mistaken for "unset" and overridden by the pipeline's model default. This is
|
||||
# the sentinel collision that the None default + ``is not None`` check fixes.
|
||||
req = _request_with_guidance(guidance_scale=0.0)
|
||||
|
||||
assert req.sampling_params.guidance_scale_provided is True
|
||||
assert req.sampling_params.guidance_scale == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [0.999, 1.0, 5.0])
|
||||
def test_explicit_nonzero_guidance_scale_is_honored(value):
|
||||
# Any explicitly-set value (truthy or not) is honored as provided and preserved.
|
||||
req = _request_with_guidance(guidance_scale=value)
|
||||
|
||||
assert req.sampling_params.guidance_scale_provided is True
|
||||
assert req.sampling_params.guidance_scale == value
|
||||
@@ -10,8 +10,10 @@ import pytest
|
||||
import torch
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
|
||||
from vllm.v1.metrics.stats import PrefillStats, PromptTokenStats
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
from vllm_omni.data_entry_keys import CodesStruct, MetaStruct, OmniPayload, OmniPayloadStruct
|
||||
from vllm_omni.distributed.omni_connectors.adapter import construct_next_stage_streaming_input_prompt
|
||||
from vllm_omni.distributed.omni_connectors.transfer_adapter.base import OmniTransferAdapterBase
|
||||
@@ -37,8 +39,10 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None)
|
||||
external_req_id=external_req_id or req_id,
|
||||
status=status,
|
||||
prompt_token_ids=[],
|
||||
num_prompt_tokens=0,
|
||||
num_computed_tokens=0,
|
||||
num_output_placeholders=0,
|
||||
prefill_stats=None,
|
||||
additional_information=None,
|
||||
is_finished=lambda: status == RequestStatus.FINISHED_STOPPED,
|
||||
)
|
||||
@@ -485,6 +489,48 @@ def test_load_poll_ar_request_additional_information_concats_tensors(build_adapt
|
||||
assert request.additional_information["meta"]["finished"].item() is True
|
||||
|
||||
|
||||
def test_non_ar_poll_reinitializes_prefill_stats_for_later_chunks(build_adapter):
|
||||
adapter, connector = build_adapter(stage_id=2, model_mode="generation")
|
||||
request = _req("req-later-chunk", RequestStatus.WAITING, external_req_id="ext-later-chunk")
|
||||
request.prefill_stats = PrefillStats()
|
||||
adapter.request_ids_mapping[request.request_id] = request.external_req_id
|
||||
|
||||
connector.get.return_value = (
|
||||
{
|
||||
"codes": {"audio": torch.tensor([7, 8], dtype=torch.long)},
|
||||
"meta": {"finished": torch.tensor(False, dtype=torch.bool)},
|
||||
},
|
||||
8,
|
||||
)
|
||||
assert adapter._poll_single_request(request) is True
|
||||
OmniGenerationScheduler._record_prefill_stats(request)
|
||||
first_chunk_stats = request.prefill_stats
|
||||
request.prefill_stats = None
|
||||
|
||||
connector.get.return_value = (
|
||||
{
|
||||
"codes": {"audio": torch.tensor([9, 10, 11], dtype=torch.long)},
|
||||
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
|
||||
},
|
||||
8,
|
||||
)
|
||||
assert adapter._poll_single_request(request) is True
|
||||
assert isinstance(request.prefill_stats, PrefillStats)
|
||||
OmniGenerationScheduler._record_prefill_stats(request)
|
||||
second_chunk_stats = request.prefill_stats
|
||||
|
||||
assert first_chunk_stats is not None
|
||||
assert first_chunk_stats.num_prompt_tokens == 2
|
||||
assert second_chunk_stats is not None
|
||||
assert second_chunk_stats.num_prompt_tokens == 3
|
||||
|
||||
prompt_token_stats = PromptTokenStats()
|
||||
prompt_token_stats.update_from_output(first_chunk_stats)
|
||||
prompt_token_stats.update_from_output(second_chunk_stats)
|
||||
assert prompt_token_stats.total == 5
|
||||
assert prompt_token_stats.computed == 5
|
||||
|
||||
|
||||
def test_sender_only_adapter_does_not_park_or_clear_requests(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1, connector_extra={"role": "sender"})
|
||||
request = _req("req-1", RequestStatus.WAITING)
|
||||
|
||||
@@ -468,6 +468,10 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None:
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {"text": ["london"]},
|
||||
# A one-word English clip is too short for whisper's language detection to
|
||||
# be stable; unpinned it renders "London" as 런던 / ランデ / 梁敦 and the
|
||||
# containment check then fails on correct audio.
|
||||
"transcript_language": "en",
|
||||
}
|
||||
|
||||
# Retry only when assert_omni_response fails on text/audio cosine similarity (see tests/helpers/assertions.py).
|
||||
|
||||
@@ -7,7 +7,9 @@ from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm_omni.distributed.omni_coordinator import ReplicaInfo, ReplicaStatus
|
||||
from vllm_omni.engine import OmniEngineCoreRequest
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine, StageRuntimeInfo
|
||||
from vllm_omni.engine.serialization import deserialize_additional_information
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.model_executor.stage_input_processors.bagel import ExpandedPrompt
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -107,6 +109,34 @@ def test_build_add_request_message_preserves_model_intermediate_buffer(mocker: M
|
||||
assert torch.equal(info["hidden_states"]["tts"], hidden)
|
||||
|
||||
|
||||
def test_cfg_companion_suppresses_payload_but_forces_kv_transfer(mocker: MockerFixture):
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
params = SamplingParams(max_tokens=8)
|
||||
engine.prompt_expand_func = lambda *_args: [
|
||||
ExpandedPrompt(
|
||||
prompt={"prompt": "negative"},
|
||||
role="cfg_text",
|
||||
request_id_suffix="__cfg_text",
|
||||
)
|
||||
]
|
||||
engine.supported_tasks = ("generate",)
|
||||
engine.input_processor = mocker.Mock()
|
||||
engine.input_processor.process_inputs.return_value = _make_engine_core_request("req__cfg_text")
|
||||
engine.request_queue = mocker.Mock()
|
||||
|
||||
engine._enqueue_cfg_companions(
|
||||
parent_id="req",
|
||||
original_prompt={"prompt": "positive"},
|
||||
stage0_params=params,
|
||||
sampling_params_list=[params],
|
||||
)
|
||||
|
||||
message = engine.request_queue.sync_q.put.call_args.args[0]
|
||||
metadata = deserialize_additional_information(message.prompt.additional_information)
|
||||
assert metadata["omni_final_stage_id"] == 0
|
||||
assert metadata["omni_force_kv_transfer"] is True
|
||||
|
||||
|
||||
def test_build_add_request_message_with_resumable_streaming(mocker: MockerFixture):
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
params = SamplingParams(max_tokens=8)
|
||||
|
||||
@@ -172,6 +172,41 @@ def test_stage_engine_core_client_module_reload_keeps_forward_refs_deferred():
|
||||
)
|
||||
|
||||
|
||||
def test_async_omni_engine_initialize_stages_passes_log_stats_to_runtime(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.stage_configs = [types.SimpleNamespace()]
|
||||
engine.model = "dummy-model"
|
||||
engine.config_path = "dummy-config"
|
||||
engine.single_stage_mode = False
|
||||
engine.diffusion_batch_size = 1
|
||||
engine.async_chunk = False
|
||||
engine.tokenizer = None
|
||||
engine._single_stage_id_filter = None
|
||||
engine._omni_master_address = None
|
||||
engine._omni_master_port = None
|
||||
engine._omni_dp_size_local = 1
|
||||
engine._omni_heartbeat_timeout = 30.0
|
||||
engine._omni_lb_policy = "random"
|
||||
engine.request_queue = types.SimpleNamespace()
|
||||
engine._log_stats = True
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
runtime = types.SimpleNamespace(stage_pools=[], initialize=lambda: None)
|
||||
|
||||
def _capture_create_stage_runtime(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return runtime
|
||||
|
||||
monkeypatch.setattr(engine_mod, "create_stage_runtime", _capture_create_stage_runtime)
|
||||
|
||||
engine._initialize_stages(stage_init_timeout=7)
|
||||
|
||||
assert captured["stage_init_timeout"] == 7
|
||||
assert captured["log_stats"] is True
|
||||
|
||||
|
||||
def test_compute_replica_layout_splits_diffusion_devices_by_world_size():
|
||||
stage_cfg = types.SimpleNamespace(
|
||||
stage_id=0,
|
||||
@@ -342,6 +377,95 @@ def test_stage_runtime_initializes_stage_pools(monkeypatch):
|
||||
assert runtime.stage_pools[1].output_processor is stage1_output_processor
|
||||
|
||||
|
||||
def test_stage_runtime_passes_log_stats_to_llm_replica_launch(monkeypatch):
|
||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||
|
||||
runtime = StageRuntime(
|
||||
stage_configs=[types.SimpleNamespace()],
|
||||
model="dummy-model",
|
||||
config_path="dummy-config",
|
||||
stage_init_timeout=1,
|
||||
diffusion_batch_size=1,
|
||||
async_chunk=False,
|
||||
log_stats=True,
|
||||
)
|
||||
cfg = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
plan = _make_llm_plan(0, stage_id=0, vllm_config=cfg).replicas[0]
|
||||
plan.engine_args_dict = {}
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
addresses = types.SimpleNamespace(
|
||||
inputs=["tcp://127.0.0.1:1"],
|
||||
outputs=["tcp://127.0.0.1:2"],
|
||||
frontend_stats_publish_address=None,
|
||||
)
|
||||
resources = types.SimpleNamespace(manager=object(), coordinator=None, addresses=addresses)
|
||||
stage_client = types.SimpleNamespace()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _capture_launch_stage_replica(**kwargs):
|
||||
captured["log_stats"] = kwargs["log_stats"]
|
||||
yield resources
|
||||
|
||||
monkeypatch.setattr(runtime_mod, "acquire_device_locks", lambda *_args, **_kwargs: [])
|
||||
monkeypatch.setattr(runtime_mod, "launch_stage_replica", _capture_launch_stage_replica)
|
||||
monkeypatch.setattr(
|
||||
runtime_mod.StageEngineCoreClientBase,
|
||||
"make_async_mp_client",
|
||||
lambda **kwargs: (captured.__setitem__("client_log_stats", kwargs["log_stats"]) or stage_client),
|
||||
)
|
||||
|
||||
assert runtime._initialize_local_llm_replica(plan, stage_init_timeout=1) is stage_client
|
||||
assert captured["log_stats"] is True
|
||||
assert captured["client_log_stats"] is True
|
||||
|
||||
|
||||
def test_stage_runtime_passes_log_stats_to_output_processor(monkeypatch):
|
||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||
|
||||
runtime = StageRuntime(
|
||||
stage_configs=[types.SimpleNamespace()],
|
||||
model="dummy-model",
|
||||
config_path="dummy-config",
|
||||
stage_init_timeout=1,
|
||||
diffusion_batch_size=1,
|
||||
async_chunk=False,
|
||||
log_stats=True,
|
||||
)
|
||||
cfg = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
stage_plan = _make_llm_plan(0, stage_id=0, vllm_config=cfg)
|
||||
stage_client = types.SimpleNamespace(
|
||||
stage_type="llm",
|
||||
is_comprehension=False,
|
||||
final_output=True,
|
||||
final_output_type=None,
|
||||
default_sampling_params=types.SimpleNamespace(),
|
||||
)
|
||||
captured: dict[str, object] = {}
|
||||
output_processor = object()
|
||||
|
||||
def _capture_build_llm_stage_output_processor(plan, stage_vllm_config, *, log_stats=False):
|
||||
captured["plan"] = plan
|
||||
captured["stage_vllm_config"] = stage_vllm_config
|
||||
captured["log_stats"] = log_stats
|
||||
return output_processor
|
||||
|
||||
monkeypatch.setattr(
|
||||
runtime_mod,
|
||||
"build_llm_stage_output_processor",
|
||||
_capture_build_llm_stage_output_processor,
|
||||
)
|
||||
|
||||
pools = runtime._assemble_stage_pools([stage_plan], {0: [stage_client]})
|
||||
|
||||
assert pools[0].output_processor is output_processor
|
||||
assert captured == {
|
||||
"plan": stage_plan,
|
||||
"stage_vllm_config": cfg,
|
||||
"log_stats": True,
|
||||
}
|
||||
|
||||
|
||||
def test_build_logical_stage_init_plans_applies_replica_device_splits(monkeypatch):
|
||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||
|
||||
@@ -537,10 +661,11 @@ def test_initialize_local_llm_replica_passes_stage_init_timeout_to_complete_stag
|
||||
stage_configs=[],
|
||||
model="dummy-model",
|
||||
config_path="dummy-config",
|
||||
stage_init_timeout=302,
|
||||
stage_init_timeout=1,
|
||||
diffusion_batch_size=1,
|
||||
async_chunk=False,
|
||||
)
|
||||
stage_init_timeout = 302
|
||||
|
||||
fake_vllm_config = types.SimpleNamespace()
|
||||
fake_addresses = types.SimpleNamespace(inputs=["in"], outputs=["out"], frontend_stats_publish_address=None)
|
||||
@@ -587,14 +712,14 @@ def test_initialize_local_llm_replica_passes_stage_init_timeout_to_complete_stag
|
||||
)
|
||||
|
||||
try:
|
||||
runtime._initialize_local_llm_replica(plan, 302)
|
||||
runtime._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
finally:
|
||||
if prev_device_env is None:
|
||||
os.environ.pop(device_env_var, None)
|
||||
else:
|
||||
os.environ[device_env_var] = prev_device_env
|
||||
|
||||
assert captured_timeout == 302
|
||||
assert captured_timeout == stage_init_timeout
|
||||
|
||||
|
||||
def test_build_engine_args_cli_tokenizer_overrides_inferred_base_tokenizer(tmp_path):
|
||||
|
||||
@@ -15,6 +15,7 @@ import pytest
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
|
||||
from vllm_omni.engine.messages import (
|
||||
AbortRequestMessage,
|
||||
@@ -145,7 +146,7 @@ class FakeStageClient:
|
||||
try:
|
||||
return self._engine_core_outputs.get_nowait()
|
||||
except queue.Empty:
|
||||
return SimpleNamespace(outputs=[])
|
||||
return SimpleNamespace(outputs=[], scheduler_stats=None, finished_requests=None)
|
||||
|
||||
def get_diffusion_output_nowait(self):
|
||||
try:
|
||||
@@ -253,12 +254,40 @@ class FakeOutputProcessor:
|
||||
return None
|
||||
|
||||
|
||||
class IterationStatsOutputProcessor(FakeOutputProcessor):
|
||||
def process_outputs(self, *_args, **kwargs):
|
||||
iteration_stats = kwargs.get("iteration_stats")
|
||||
if iteration_stats is None and len(_args) >= 3:
|
||||
iteration_stats = _args[2]
|
||||
assert iteration_stats is not None
|
||||
iteration_stats.num_generation_tokens += 3
|
||||
return super().process_outputs(*_args, **kwargs)
|
||||
|
||||
|
||||
class RecordingOutputProcessor(FakeOutputProcessor):
|
||||
def __init__(self, *, request_outputs: list[object] | None = None) -> None:
|
||||
super().__init__(request_outputs=request_outputs)
|
||||
self.process_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
|
||||
|
||||
def process_outputs(self, *args, **kwargs):
|
||||
self.process_calls.append((args, kwargs))
|
||||
return super().process_outputs(*args, **kwargs)
|
||||
|
||||
|
||||
class RecordingStatLogger:
|
||||
def __init__(self) -> None:
|
||||
self.records: list[tuple[Any, Any, int]] = []
|
||||
|
||||
def record(self, scheduler_stats, iteration_stats, *, engine_idx: int = 0) -> None:
|
||||
self.records.append((scheduler_stats, iteration_stats, engine_idx))
|
||||
|
||||
|
||||
def _sampling_params(max_tokens: int = 4) -> SamplingParams:
|
||||
return SamplingParams(max_tokens=max_tokens)
|
||||
|
||||
|
||||
def _engine_core_outputs(tag: str, timestamp: float) -> SimpleNamespace:
|
||||
return SimpleNamespace(outputs=[tag], timestamp=timestamp, scheduler_stats=None)
|
||||
return SimpleNamespace(outputs=[tag], timestamp=timestamp, scheduler_stats=None, finished_requests=None)
|
||||
|
||||
|
||||
def _build_request_output(
|
||||
@@ -329,6 +358,7 @@ def _build_harness(
|
||||
output_processors: list[object] | None = None,
|
||||
stage_vllm_configs: list[object] | None = None,
|
||||
async_chunk: bool = False,
|
||||
log_stats: bool = False,
|
||||
stage_pools: list[StagePool] | None = None,
|
||||
) -> OrchestratorFixture:
|
||||
"""Build an Orchestrator test harness.
|
||||
@@ -364,6 +394,7 @@ def _build_harness(
|
||||
rpc_async_queue=rpc_queue.async_q,
|
||||
stage_pools=stage_pools,
|
||||
async_chunk=async_chunk,
|
||||
log_stats=log_stats,
|
||||
)
|
||||
ready_future.set_result((orchestrator, request_queue, output_queue, rpc_queue))
|
||||
await orchestrator.run()
|
||||
@@ -2137,38 +2168,143 @@ async def test_multi_replica_cfg_companion_inherits_parent_affinity(orchestrator
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
def test_orchestrator_does_not_re_introduce_global_stats_throttle() -> None:
|
||||
"""Regression: each (stage, replica) must independently publish its wrapped
|
||||
vllm:* stats when its scheduler emits non-None scheduler_stats.
|
||||
|
||||
A previous version of Orchestrator carried a global self._last_stats_ts /
|
||||
_stats_interval_s gate around _stat_logger.record(). Because
|
||||
OmniSchedulerMixin.make_stats() already throttles at 1 Hz per scheduler
|
||||
(one per (stage, replica)), the extra global gate starved every replica
|
||||
other than the first to emit within each second — their {stage, replica}
|
||||
gauges/counters went stale.
|
||||
|
||||
The fix removed the global gate entirely; the only signal needed is
|
||||
'this replica's scheduler emitted non-None scheduler_stats'. This test
|
||||
fails loudly if someone reintroduces the global throttle.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
from vllm_omni.engine.orchestrator import Orchestrator
|
||||
|
||||
source = inspect.getsource(Orchestrator)
|
||||
assert "_last_stats_ts" not in source, (
|
||||
"Orchestrator must not gate stat recording on a global timestamp. "
|
||||
"OmniSchedulerMixin.make_stats() already throttles per scheduler "
|
||||
"(per (stage, replica)); an outer global gate starves all but the "
|
||||
"first replica to emit within each 1s window."
|
||||
@pytest.mark.asyncio
|
||||
async def test_orchestrator_records_iteration_stats_without_scheduler_stats(orchestrator_factory) -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
processor = IterationStatsOutputProcessor(
|
||||
request_outputs=[_build_request_output("req-stats", token_ids=[1], finished=True)]
|
||||
)
|
||||
assert "_stats_interval_s" not in source
|
||||
assert "raw_outputs.scheduler_stats is not None" in source, (
|
||||
"Orchestrator must gate stat recording solely on "
|
||||
"raw_outputs.scheduler_stats being non-None — the per-scheduler 1Hz "
|
||||
"throttle in OmniSchedulerMixin.make_stats() is the only gate needed."
|
||||
orchestrator_fixture = orchestrator_factory([stage0], output_processors=[processor], log_stats=True)
|
||||
stat_logger = RecordingStatLogger()
|
||||
orchestrator_fixture.orchestrator._stat_logger = stat_logger
|
||||
request = SimpleNamespace(request_id="req-stats", prompt_token_ids=[1, 2])
|
||||
|
||||
try:
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="req-stats",
|
||||
prompt=request,
|
||||
original_prompt={"prompt": "hello"},
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
await _wait_for(lambda: len(stage0.add_request_calls) == 1)
|
||||
|
||||
stage0.push_engine_core_outputs(_engine_core_outputs("raw-without-scheduler-stats", 1.0))
|
||||
|
||||
await _wait_for(lambda: len(stat_logger.records) == 1)
|
||||
scheduler_stats, iteration_stats, engine_idx = stat_logger.records[0]
|
||||
assert scheduler_stats is None
|
||||
assert iteration_stats is not None
|
||||
assert iteration_stats.num_generation_tokens == 3
|
||||
assert engine_idx == 0
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orchestrator_records_scheduler_stats_without_outputs(orchestrator_factory) -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
processor = RecordingOutputProcessor()
|
||||
orchestrator_fixture = orchestrator_factory([stage0], output_processors=[processor], log_stats=True)
|
||||
stat_logger = RecordingStatLogger()
|
||||
orchestrator_fixture.orchestrator._stat_logger = stat_logger
|
||||
scheduler_stats = SimpleNamespace(num_running_reqs=0, kv_cache_usage=0.5)
|
||||
|
||||
try:
|
||||
stage0.push_engine_core_outputs(
|
||||
SimpleNamespace(
|
||||
outputs=[],
|
||||
timestamp=1.0,
|
||||
scheduler_stats=scheduler_stats,
|
||||
finished_requests=None,
|
||||
)
|
||||
)
|
||||
|
||||
await _wait_for(lambda: len(stat_logger.records) == 1)
|
||||
recorded_scheduler_stats, iteration_stats, engine_idx = stat_logger.records[0]
|
||||
assert recorded_scheduler_stats is scheduler_stats
|
||||
assert iteration_stats is None
|
||||
assert engine_idx == 0
|
||||
assert processor.process_calls[0][0][2] is None
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orchestrator_does_not_build_iteration_stats_for_finished_only_batch(orchestrator_factory) -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
processor = RecordingOutputProcessor()
|
||||
orchestrator_fixture = orchestrator_factory([stage0], output_processors=[processor], log_stats=True)
|
||||
stat_logger = RecordingStatLogger()
|
||||
orchestrator_fixture.orchestrator._stat_logger = stat_logger
|
||||
|
||||
try:
|
||||
stage0.push_engine_core_outputs(
|
||||
SimpleNamespace(
|
||||
outputs=[],
|
||||
timestamp=1.0,
|
||||
scheduler_stats=None,
|
||||
finished_requests={"req-finished"},
|
||||
)
|
||||
)
|
||||
|
||||
await _wait_for(lambda: bool(processor.process_calls))
|
||||
assert processor.process_calls[0][0][2] is None
|
||||
assert stat_logger.records == []
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orchestrator_does_not_build_iteration_stats_without_stat_logger(orchestrator_factory) -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
processor = RecordingOutputProcessor()
|
||||
orchestrator_fixture = orchestrator_factory([stage0], output_processors=[processor], log_stats=True)
|
||||
orchestrator_fixture.orchestrator._stat_logger = None
|
||||
|
||||
try:
|
||||
stage0.push_engine_core_outputs(_engine_core_outputs("raw-output", 1.0))
|
||||
|
||||
await _wait_for(lambda: bool(processor.process_calls))
|
||||
assert processor.process_calls[0][0][2] is None
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage_pool_preserves_none_iteration_stats() -> None:
|
||||
client = FakeStageClient(stage_type="llm", final_output=True)
|
||||
processor = RecordingOutputProcessor()
|
||||
pool = StagePool(
|
||||
0,
|
||||
[client],
|
||||
output_processor=processor,
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
)
|
||||
raw_outputs = SimpleNamespace(outputs=["raw"], timestamp=1.0, scheduler_stats=None)
|
||||
|
||||
await pool.process_llm_raw_outputs(0, raw_outputs, iteration_stats=None)
|
||||
|
||||
assert processor.process_calls[0][0][2] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage_pool_process_llm_raw_outputs_mutates_iteration_stats() -> None:
|
||||
client = FakeStageClient(stage_type="llm", final_output=True)
|
||||
processor = IterationStatsOutputProcessor()
|
||||
pool = StagePool(
|
||||
0,
|
||||
[client],
|
||||
output_processor=processor,
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
)
|
||||
raw_outputs = SimpleNamespace(outputs=["raw"], timestamp=1.0, scheduler_stats=None)
|
||||
iteration_stats = IterationStats()
|
||||
|
||||
await pool.process_llm_raw_outputs(0, raw_outputs, iteration_stats=iteration_stats)
|
||||
|
||||
assert iteration_stats.num_generation_tokens == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -10,9 +10,11 @@ import pytest
|
||||
import torch
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.engine import EngineCoreEvent, EngineCoreEventType, FinishReason
|
||||
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
|
||||
from vllm.v1.metrics.stats import IterationStats, PrefillStats
|
||||
|
||||
from vllm_omni.engine import OmniEngineCoreOutput
|
||||
from vllm_omni.outputs import output_processor
|
||||
from vllm_omni.outputs.output_modality import OutputModality, OutputModalityNames
|
||||
from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState
|
||||
@@ -616,3 +618,48 @@ def test_no_detokenizer_final_only():
|
||||
result = s.make_request_output([], None, FinishReason.STOP, None)
|
||||
assert result is not None
|
||||
assert AUDIO in result.outputs[0].multimodal_output
|
||||
|
||||
|
||||
def test_mm_only_outputs_update_iteration_stats():
|
||||
processor = MultimodalOutputProcessor(
|
||||
tokenizer=None,
|
||||
log_stats=True,
|
||||
engine_core_output_type=AUDIO,
|
||||
)
|
||||
state = _make_no_detok_state(RequestOutputKind.CUMULATIVE)
|
||||
processor.request_states[state.request_id] = state
|
||||
processor.external_req_ids[state.external_req_id].append(state.request_id)
|
||||
|
||||
prefill_stats = PrefillStats()
|
||||
prefill_stats.set(
|
||||
num_prompt_tokens=4,
|
||||
num_local_cached_tokens=0,
|
||||
num_external_cached_tokens=0,
|
||||
)
|
||||
output = OmniEngineCoreOutput(
|
||||
request_id=state.request_id,
|
||||
new_token_ids=[10, 11],
|
||||
finish_reason=FinishReason.STOP,
|
||||
events=[
|
||||
EngineCoreEvent(EngineCoreEventType.QUEUED, 1.0),
|
||||
EngineCoreEvent(EngineCoreEventType.SCHEDULED, 2.0),
|
||||
],
|
||||
prefill_stats=prefill_stats,
|
||||
multimodal_output={AUDIO: torch.ones(1, 8)},
|
||||
)
|
||||
iteration_stats = IterationStats()
|
||||
|
||||
processor.process_outputs(
|
||||
[output],
|
||||
engine_core_timestamp=3.0,
|
||||
iteration_stats=iteration_stats,
|
||||
)
|
||||
|
||||
assert iteration_stats.num_prompt_tokens == 4
|
||||
assert iteration_stats.num_generation_tokens == 2
|
||||
assert len(iteration_stats.time_to_first_tokens_iter) == 1
|
||||
assert len(iteration_stats.finished_requests) == 1
|
||||
finished = iteration_stats.finished_requests[0]
|
||||
assert finished.finish_reason == FinishReason.STOP
|
||||
assert finished.num_prompt_tokens == state.prompt_len
|
||||
assert finished.num_generation_tokens == 2
|
||||
@@ -778,6 +778,7 @@ class TestSingleStageReplicaInitialization:
|
||||
single_stage_id_filter=None,
|
||||
omni_master_address="127.0.0.1",
|
||||
omni_master_port=26000,
|
||||
log_stats=True,
|
||||
)
|
||||
runtime._omni_master_server = mocker.Mock(spec=OmniMasterServer)
|
||||
runtime._omni_master_server.get_stage_config.return_value = {"stage_id": 1, "stage_type": "llm"}
|
||||
@@ -811,10 +812,17 @@ class TestSingleStageReplicaInitialization:
|
||||
sentinel_client = SimpleNamespace()
|
||||
|
||||
mock_connect = mocker.patch.object(runtime_mod, "connect_remote_engine_cores", side_effect=_fake_connect)
|
||||
client_kwargs: dict[str, Any] = {}
|
||||
|
||||
def _capture_make_async_mp_client(**kwargs):
|
||||
client_kwargs.update(kwargs)
|
||||
events.append("attach")
|
||||
return sentinel_client
|
||||
|
||||
mocker.patch.object(
|
||||
StageEngineCoreClientBase,
|
||||
"make_async_mp_client",
|
||||
side_effect=lambda **_: (events.append("attach"), sentinel_client)[1],
|
||||
side_effect=_capture_make_async_mp_client,
|
||||
)
|
||||
|
||||
result = runtime._initialize_remote_replica(plan, stage_init_timeout=60)
|
||||
@@ -824,6 +832,7 @@ class TestSingleStageReplicaInitialization:
|
||||
assert mock_connect.call_args.kwargs["vllm_config"].parallel_config.data_parallel_size_local == 0
|
||||
assert mock_connect.call_args.kwargs["stage_id"] == 1
|
||||
assert mock_connect.call_args.kwargs["replica_id"] == 0
|
||||
assert client_kwargs["log_stats"] is True
|
||||
assert events == ["enter", "exit", "attach"]
|
||||
|
||||
def test_initialize_llm_replica_remote_missing_registered_stage_config_raises(self, mocker: MockerFixture):
|
||||
|
||||
@@ -1301,7 +1301,11 @@ def test_generate_images_rejects_model_mismatch(test_client):
|
||||
assert "model mismatch" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
def test_image_file_response_format_multiple(test_client):
|
||||
@pytest.mark.parametrize(
|
||||
("output_format", "expected_pil_format"),
|
||||
[("png", "PNG"), ("jpeg", "JPEG"), ("webp", "WEBP")],
|
||||
)
|
||||
def test_image_file_response_format_multiple(test_client, output_format, expected_pil_format):
|
||||
"""Test response_format=file with n>1 returns ZIP archive"""
|
||||
response = test_client.post(
|
||||
"/v1/images/generations",
|
||||
@@ -1309,6 +1313,7 @@ def test_image_file_response_format_multiple(test_client):
|
||||
"prompt": "a dog",
|
||||
"n": 3,
|
||||
"response_format": "file",
|
||||
"output_format": output_format,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1317,23 +1322,32 @@ def test_image_file_response_format_multiple(test_client):
|
||||
assert "attachment" in response.headers.get("content-disposition", "")
|
||||
assert ".zip" in response.headers.get("content-disposition", "")
|
||||
|
||||
# Verify it's a valid ZIP with 3 PNG files
|
||||
# Verify it's a valid ZIP with 3 correctly labelled image files
|
||||
import zipfile
|
||||
|
||||
zip_buffer = io.BytesIO(response.content)
|
||||
with zipfile.ZipFile(zip_buffer, "r") as zf:
|
||||
files = zf.namelist()
|
||||
assert len(files) == 3
|
||||
assert all(f.endswith(".png") for f in files)
|
||||
assert all(f.endswith(f".{output_format}") for f in files)
|
||||
|
||||
# Verify each file is a valid PNG
|
||||
# Verify each file's bytes match its advertised format
|
||||
for filename in files:
|
||||
img_bytes = zf.read(filename)
|
||||
img = Image.open(io.BytesIO(img_bytes))
|
||||
assert img.format == "PNG"
|
||||
assert img.format == expected_pil_format
|
||||
|
||||
|
||||
def test_image_file_response_format_single(test_client):
|
||||
@pytest.mark.parametrize(
|
||||
("output_format", "expected_media_type", "expected_pil_format"),
|
||||
[
|
||||
("png", "image/png", "PNG"),
|
||||
("jpg", "image/jpeg", "JPEG"),
|
||||
("jpeg", "image/jpeg", "JPEG"),
|
||||
("webp", "image/webp", "WEBP"),
|
||||
],
|
||||
)
|
||||
def test_image_file_response_format_single(test_client, output_format, expected_media_type, expected_pil_format):
|
||||
"""Test response_format=file with n=1 returns a single image file."""
|
||||
response = test_client.post(
|
||||
"/v1/images/generations",
|
||||
@@ -1341,16 +1355,61 @@ def test_image_file_response_format_single(test_client):
|
||||
"prompt": "a dog",
|
||||
"n": 1,
|
||||
"response_format": "file",
|
||||
"output_format": output_format,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"] == "image/png"
|
||||
assert response.headers["content-type"] == expected_media_type
|
||||
assert "attachment" in response.headers.get("content-disposition", "")
|
||||
assert ".png" in response.headers.get("content-disposition", "")
|
||||
assert f".{output_format}" in response.headers.get("content-disposition", "")
|
||||
|
||||
img = Image.open(io.BytesIO(response.content))
|
||||
assert img.format == "PNG"
|
||||
assert img.format == expected_pil_format
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("output_format", "image_count", "expected_media_type", "expected_pil_format"),
|
||||
[
|
||||
("jpeg", 1, "image/jpeg", "JPEG"),
|
||||
("webp", 2, "application/zip", "WEBP"),
|
||||
],
|
||||
)
|
||||
def test_multistage_image_file_response_uses_requested_format(
|
||||
async_omni_test_client,
|
||||
output_format,
|
||||
image_count,
|
||||
expected_media_type,
|
||||
expected_pil_format,
|
||||
):
|
||||
engine = async_omni_test_client.app.state.engine_client
|
||||
engine._images = [Image.new("RGB", (16, 16), color="green") for _ in range(image_count)]
|
||||
|
||||
response = async_omni_test_client.post(
|
||||
"/v1/images/generations",
|
||||
json={
|
||||
"prompt": "a dog",
|
||||
"n": image_count,
|
||||
"response_format": "file",
|
||||
"output_format": output_format,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"] == expected_media_type
|
||||
if image_count == 1:
|
||||
assert f".{output_format}" in response.headers["content-disposition"]
|
||||
assert Image.open(io.BytesIO(response.content)).format == expected_pil_format
|
||||
return
|
||||
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(response.content), "r") as archive:
|
||||
filenames = archive.namelist()
|
||||
assert len(filenames) == image_count
|
||||
assert all(filename.endswith(f".{output_format}") for filename in filenames)
|
||||
for filename in filenames:
|
||||
assert Image.open(io.BytesIO(archive.read(filename))).format == expected_pil_format
|
||||
|
||||
|
||||
def make_test_image_bytes(size=(64, 64)) -> bytes:
|
||||
|
||||
@@ -48,6 +48,25 @@ def test_encode_video_bytes_exports_frames_without_interpolation(monkeypatch):
|
||||
assert mux_calls[0]["audio"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("frame_count", [3, 4])
|
||||
def test_channel_last_video_tensor_preserves_channel_sized_frame_count(frame_count):
|
||||
video = torch.arange(frame_count * 2 * 5 * 3, dtype=torch.uint8).reshape(frame_count, 2, 5, 3)
|
||||
|
||||
frames = video_api_utils._coerce_video_to_uint8_frames(video)
|
||||
|
||||
assert frames.shape == (frame_count, 2, 5, 3)
|
||||
np.testing.assert_array_equal(frames, video.numpy())
|
||||
|
||||
|
||||
def test_channel_first_video_tensor_is_converted_to_channel_last():
|
||||
video = torch.arange(3 * 5 * 2 * 6, dtype=torch.uint8).reshape(3, 5, 2, 6)
|
||||
|
||||
frames = video_api_utils._coerce_video_to_uint8_frames(video)
|
||||
|
||||
assert frames.shape == (5, 2, 6, 3)
|
||||
np.testing.assert_array_equal(frames, video.permute(1, 2, 3, 0).numpy())
|
||||
|
||||
|
||||
def test_fragmented_mp4_video_encoder_reuses_single_muxer(monkeypatch):
|
||||
muxers = []
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ Key invariants tested:
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -80,28 +81,37 @@ def _make_audio_omni_output(
|
||||
request_id: str = "test-req",
|
||||
index: int = 0,
|
||||
num_prompt_tokens: int = 3,
|
||||
stage_id: int | None = None,
|
||||
replica_id: int | None = None,
|
||||
audio_samples: int = 0,
|
||||
) -> OmniRequestOutput:
|
||||
"""Build an OmniRequestOutput for audio (no torch dependency)."""
|
||||
completion = CompletionOutput(
|
||||
index=index,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
)
|
||||
if audio_samples > 0:
|
||||
completion.multimodal_output = {
|
||||
"audio": [MagicMock(numel=MagicMock(return_value=audio_samples))],
|
||||
"sr": 24000,
|
||||
}
|
||||
res = RequestOutput(
|
||||
request_id=request_id,
|
||||
prompt="test",
|
||||
prompt_token_ids=list(range(num_prompt_tokens)),
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=index,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
)
|
||||
],
|
||||
outputs=[completion],
|
||||
finished=True,
|
||||
)
|
||||
return OmniRequestOutput(
|
||||
request_id=request_id,
|
||||
stage_id=stage_id,
|
||||
replica_id=replica_id,
|
||||
final_output_type="audio",
|
||||
request_output=res,
|
||||
finished=True,
|
||||
@@ -321,6 +331,74 @@ async def test_single_modality_audio_only_one_stop():
|
||||
assert finish_reasons[-1] == "stop"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("output_replica_id", "pool_replica_id", "expected_replica_id", "pool_lookup"),
|
||||
[
|
||||
(7, None, 7, False),
|
||||
(None, 5, 5, True),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_audio_metrics_resolve_replica_id(
|
||||
monkeypatch,
|
||||
output_replica_id,
|
||||
pool_replica_id,
|
||||
expected_replica_id,
|
||||
pool_lookup,
|
||||
):
|
||||
import vllm_omni.entrypoints.openai.serving_chat as serving_chat_mod
|
||||
from vllm_omni.entrypoints.client_request_state import ClientRequestState
|
||||
|
||||
serving_chat = _build_serving_chat()
|
||||
request = _make_request(modalities=["audio"])
|
||||
req_state = ClientRequestState(
|
||||
request_id="internal-req",
|
||||
external_request_id="test-req",
|
||||
)
|
||||
req_state.request_arrival_ts = time.time() - 1.0
|
||||
serving_chat.engine_client.request_states = {"internal-req": req_state}
|
||||
serving_chat.engine_client.mod_metrics = object()
|
||||
serving_chat.engine_client.engine = MagicMock()
|
||||
serving_chat.engine_client.engine.stage_pools = [MagicMock(), MagicMock(), MagicMock()]
|
||||
serving_chat.engine_client.engine.stage_pools[2].get_bound_replica_id.return_value = pool_replica_id
|
||||
|
||||
first_packet_calls = []
|
||||
finalize_calls = []
|
||||
|
||||
def fake_observe_audio_first_packet(*args, **kwargs):
|
||||
first_packet_calls.append(kwargs)
|
||||
|
||||
def fake_observe_audio_streaming_finalize(*args, **kwargs):
|
||||
finalize_calls.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(serving_chat_mod, "observe_audio_first_packet", fake_observe_audio_first_packet)
|
||||
monkeypatch.setattr(serving_chat_mod, "observe_audio_streaming_finalize", fake_observe_audio_streaming_finalize)
|
||||
|
||||
async def result_generator():
|
||||
yield _make_audio_omni_output(stage_id=2, replica_id=output_replica_id, audio_samples=2400)
|
||||
|
||||
await _collect_stream(
|
||||
serving_chat.chat_completion_stream_generator(
|
||||
request=request,
|
||||
result_generator=result_generator(),
|
||||
request_id="test-req",
|
||||
model_name="test-model",
|
||||
conversation=[],
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=MagicMock(),
|
||||
)
|
||||
)
|
||||
|
||||
if pool_lookup:
|
||||
serving_chat.engine_client.engine.stage_pools[2].get_bound_replica_id.assert_called_once_with("internal-req")
|
||||
else:
|
||||
serving_chat.engine_client.engine.stage_pools[2].get_bound_replica_id.assert_not_called()
|
||||
assert first_packet_calls[0]["stage_id"] == 2
|
||||
assert first_packet_calls[0]["replica_id"] == expected_replica_id
|
||||
assert finalize_calls[0]["stage_id"] == 2
|
||||
assert finalize_calls[0]["replica_id"] == expected_replica_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: fallback stop chunk when declared modality is not produced
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -527,7 +527,7 @@ def _resolve_audio_transcript(
|
||||
audio_bytes = getattr(response, "audio_bytes", None)
|
||||
if not audio_bytes:
|
||||
return None
|
||||
return convert_audio_bytes_to_text(audio_bytes)
|
||||
return convert_audio_bytes_to_text(audio_bytes, language=request_config.get("transcript_language"))
|
||||
|
||||
|
||||
def assert_omni_response(response: Any, request_config: dict[str, Any], run_level):
|
||||
@@ -632,6 +632,7 @@ def _assert_transcript_matches(
|
||||
*,
|
||||
threshold: float,
|
||||
escalation_model: str | None = None,
|
||||
language: str | None = None,
|
||||
) -> None:
|
||||
"""Assert spoken audio matches ``expected_text``.
|
||||
|
||||
@@ -647,7 +648,8 @@ def _assert_transcript_matches(
|
||||
clip is re-transcribed with that stronger ASR and the assertion is decided on
|
||||
its verdict -- so a weak whisper-``small`` mishear on a short clip does not
|
||||
flake the gate, while a genuine model artifact still fails (the strong ASR
|
||||
mismatches too).
|
||||
mismatches too). ``language`` applies to that escalated pass too, so a request
|
||||
that pins a language does not silently fall back to auto-detection on retry.
|
||||
"""
|
||||
expected = str(expected_text).strip().lower()
|
||||
similarity = cosine_similarity_text(transcript.strip().lower(), expected)
|
||||
@@ -664,7 +666,7 @@ def _assert_transcript_matches(
|
||||
f"whisper-small below threshold ({similarity:.2f} <= {threshold}); "
|
||||
f"escalating to whisper-{escalation_model} to rule out an ASR mishear"
|
||||
)
|
||||
strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model)
|
||||
strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model, language=language)
|
||||
strong_similarity = cosine_similarity_text(strong_transcript.strip().lower(), expected)
|
||||
print(
|
||||
f"audio content (whisper-{escalation_model}): {strong_transcript}\n"
|
||||
@@ -741,6 +743,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
|
||||
expected_text,
|
||||
threshold=0.9,
|
||||
escalation_model=request_config.get("transcript_escalation_model"),
|
||||
language=request_config.get("transcript_language"),
|
||||
)
|
||||
_assert_preset_voice_gender_from_audio(
|
||||
response.audio_bytes,
|
||||
|
||||
+10
-5
@@ -652,7 +652,9 @@ def _serialize_whisper_model_download(model_size: str = "small"):
|
||||
f.close()
|
||||
|
||||
|
||||
def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "small") -> str:
|
||||
def _whisper_transcribe_in_current_process(
|
||||
output_path: str, model_size: str = "small", language: str | None = None
|
||||
) -> str:
|
||||
import whisper
|
||||
|
||||
device_index = None
|
||||
@@ -684,6 +686,9 @@ def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "
|
||||
temperature=0.0,
|
||||
word_timestamps=True,
|
||||
condition_on_previous_text=False,
|
||||
# None keeps whisper's auto-detection. Do not default this to a
|
||||
# language: callers include non-English audio tests.
|
||||
language=language,
|
||||
)["text"]
|
||||
finally:
|
||||
del model
|
||||
@@ -694,15 +699,15 @@ def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "
|
||||
return text or ""
|
||||
|
||||
|
||||
def convert_audio_file_to_text(output_path: str, model_size: str = "small") -> str:
|
||||
def convert_audio_file_to_text(output_path: str, model_size: str = "small", language: str | None = None) -> str:
|
||||
"""Convert an audio file to text in an isolated subprocess."""
|
||||
ctx = multiprocessing.get_context("spawn")
|
||||
with concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=ctx) as executor:
|
||||
future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size)
|
||||
future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size, language)
|
||||
return future.result()
|
||||
|
||||
|
||||
def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") -> str:
|
||||
def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small", language: str | None = None) -> str:
|
||||
output_fd, output_path = tempfile.mkstemp(prefix="test_", suffix=".wav")
|
||||
os.close(output_fd)
|
||||
if os.environ.get("VLLM_OMNI_KEEP_REQUEST_MEDIA", "").lower() not in ("1", "true", "yes"):
|
||||
@@ -710,7 +715,7 @@ def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") ->
|
||||
data, samplerate = sf.read(io.BytesIO(raw_bytes))
|
||||
sf.write(output_path, data, samplerate, format="WAV", subtype="PCM_16")
|
||||
print(f"audio data is saved: {output_path}")
|
||||
return convert_audio_file_to_text(output_path, model_size)
|
||||
return convert_audio_file_to_text(output_path, model_size, language)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
@@ -24,6 +24,19 @@ DEPLOY_CONFIG = modify_stage_config(
|
||||
1: {"kv_cache_memory_bytes": 512 * 1024 * 1024},
|
||||
2: {"kv_cache_memory_bytes": 256 * 1024 * 1024},
|
||||
},
|
||||
# Platform overrides are applied after ordinary stage settings. Keep
|
||||
# the constrained duplex test's Talker budget ahead of the base CUDA
|
||||
# profile's single-GPU default.
|
||||
"platforms": {
|
||||
"cuda": {
|
||||
"stages": [
|
||||
{
|
||||
"stage_id": 1,
|
||||
"kv_cache_memory_bytes": 512 * 1024 * 1024,
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
CORE_DEPLOY_CONFIG = modify_stage_config(
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.assertions import _assert_transcript_matches
|
||||
from tests.helpers import assertions
|
||||
from tests.helpers.assertions import _assert_transcript_matches, _resolve_audio_transcript
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -25,3 +28,63 @@ def test_short_transcript_unrelated_text_still_fails():
|
||||
expected_text="how are you",
|
||||
threshold=0.9,
|
||||
)
|
||||
|
||||
|
||||
def _capture_transcribe(monkeypatch) -> dict:
|
||||
captured: dict = {}
|
||||
|
||||
def fake_convert(raw_bytes, model_size="small", language=None):
|
||||
captured["model_size"] = model_size
|
||||
captured["language"] = language
|
||||
return "London"
|
||||
|
||||
monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", fake_convert)
|
||||
return captured
|
||||
|
||||
|
||||
def test_resolve_transcript_honors_declared_language(monkeypatch):
|
||||
captured = _capture_transcribe(monkeypatch)
|
||||
response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
|
||||
|
||||
_resolve_audio_transcript(
|
||||
response,
|
||||
{"modalities": ["text", "audio"], "transcript_language": "en"},
|
||||
"full_model",
|
||||
speech_api=False,
|
||||
)
|
||||
|
||||
assert captured["language"] == "en"
|
||||
|
||||
|
||||
def test_resolve_transcript_leaves_language_unset_by_default(monkeypatch):
|
||||
captured = _capture_transcribe(monkeypatch)
|
||||
response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
|
||||
|
||||
_resolve_audio_transcript(
|
||||
response,
|
||||
{"modalities": ["text", "audio"]},
|
||||
"full_model",
|
||||
speech_api=False,
|
||||
)
|
||||
|
||||
assert captured["language"] is None
|
||||
assert captured["model_size"] == "small"
|
||||
|
||||
|
||||
def test_escalated_transcript_keeps_declared_language(monkeypatch):
|
||||
# The escalated pass must honour the same language, otherwise a request that
|
||||
# pins one silently falls back to auto-detection on retry.
|
||||
captured = _capture_transcribe(monkeypatch)
|
||||
|
||||
with pytest.raises(AssertionError, match="after ASR escalation"):
|
||||
_assert_transcript_matches(
|
||||
"totally different words",
|
||||
audio_bytes=b"fake-wav",
|
||||
expected_text="how are you",
|
||||
threshold=0.9,
|
||||
escalation_model="large-v3",
|
||||
language="en",
|
||||
)
|
||||
|
||||
assert captured["model_size"] == "large-v3"
|
||||
assert captured["language"] == "en"
|
||||
@@ -0,0 +1,81 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
|
||||
from tests.helpers import media
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _stub_transcribe(monkeypatch) -> dict:
|
||||
"""Capture the kwargs the helper hands to whisper, with no model and no device."""
|
||||
captured: dict = {}
|
||||
|
||||
class FakeModel:
|
||||
def transcribe(self, path, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"text": "London"}
|
||||
|
||||
monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace(load_model=lambda size, device=None: FakeModel()))
|
||||
# Keep the unit test off the accelerator probe so it stays hermetic.
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"vllm_omni.platforms",
|
||||
SimpleNamespace(current_omni_platform=SimpleNamespace(is_available=lambda: False)),
|
||||
)
|
||||
return captured
|
||||
|
||||
|
||||
def test_transcribe_forwards_requested_language(monkeypatch):
|
||||
captured = _stub_transcribe(monkeypatch)
|
||||
|
||||
media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small", language="en")
|
||||
|
||||
assert captured["language"] == "en"
|
||||
|
||||
|
||||
def test_transcribe_defaults_to_auto_language(monkeypatch):
|
||||
# Auto-detect must stay the default: forcing a language globally would break
|
||||
# the non-English audio tests (e.g. the Chinese Qwen3-Omni prompts).
|
||||
captured = _stub_transcribe(monkeypatch)
|
||||
|
||||
media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small")
|
||||
|
||||
assert captured.get("language") is None
|
||||
|
||||
|
||||
def test_bytes_entrypoint_forwards_language_to_subprocess(monkeypatch, tmp_path):
|
||||
"""Cover the two hops the tests above skip: bytes -> file -> executor.submit.
|
||||
|
||||
The real call crosses a spawn ProcessPoolExecutor, so capture what gets
|
||||
submitted rather than what the worker eventually does.
|
||||
"""
|
||||
submitted: dict = {}
|
||||
|
||||
class FakeExecutor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def submit(self, fn, *args):
|
||||
submitted["fn"] = fn
|
||||
submitted["args"] = args
|
||||
return SimpleNamespace(result=lambda: "London")
|
||||
|
||||
monkeypatch.setattr(media.concurrent.futures, "ProcessPoolExecutor", lambda *a, **kw: FakeExecutor())
|
||||
|
||||
wav = tmp_path / "clip.wav"
|
||||
sf.write(wav, np.zeros(2400, dtype=np.float32), 24000)
|
||||
|
||||
assert media.convert_audio_bytes_to_text(wav.read_bytes(), "small", "en") == "London"
|
||||
|
||||
assert submitted["fn"] is media._whisper_transcribe_in_current_process
|
||||
assert submitted["args"][1:] == ("small", "en")
|
||||
@@ -170,7 +170,12 @@ class TestDeployTopology:
|
||||
0.15,
|
||||
]
|
||||
assert sum(memory_utilizations) <= 0.9 + 1e-6
|
||||
assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}}
|
||||
# Daily-Omni minicpm-interleave: up to 64 image/audio items (+ optional video).
|
||||
assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {
|
||||
"image": 64,
|
||||
"audio": 64,
|
||||
"video": 1,
|
||||
}
|
||||
elif filename in {"minicpmo_4_5_2gpu.yaml"}:
|
||||
assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [
|
||||
0.9,
|
||||
|
||||
@@ -17,6 +17,9 @@ from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.distributed.omni_connectors.kv_transfer_manager import (
|
||||
OmniKVTransferManager,
|
||||
)
|
||||
from vllm_omni.outputs import OmniConnectorOutput
|
||||
from vllm_omni.worker.omni_connector_model_runner_mixin import (
|
||||
OmniConnectorModelRunnerMixin,
|
||||
@@ -1198,6 +1201,38 @@ class TestRankAwareKVRouting(unittest.TestCase):
|
||||
)
|
||||
host.shutdown_omni_connectors()
|
||||
|
||||
def test_single_rank_hooks_use_default_manager_key(self):
|
||||
host = self._make_host(from_tp=1, to_tp=1, local_rank=0)
|
||||
expected = ["omni_0_to_1_kv_cache_req"]
|
||||
|
||||
self.assertEqual(host.get_rank_aware_kv_send_keys("req", from_stage=0, to_stage=1), expected)
|
||||
self.assertEqual(host.get_rank_aware_kv_keys("req", from_stage=0, to_stage=1), expected)
|
||||
host.shutdown_omni_connectors()
|
||||
|
||||
def test_init_uses_effective_manager_tp2_topology(self):
|
||||
model_config = _make_model_config(stage_id=0)
|
||||
model_config.omni_kv_config = {
|
||||
"need_send_cache": True,
|
||||
"omni_from_stage": "0",
|
||||
"omni_to_stage": "1",
|
||||
"rank_mapping": {"from_tp": 2, "to_tp": 2},
|
||||
}
|
||||
with patch(
|
||||
"vllm_omni.distributed.omni_connectors.kv_transfer_manager.get_local_tp_rank",
|
||||
return_value=1,
|
||||
):
|
||||
manager = OmniKVTransferManager.from_model_config(model_config)
|
||||
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(model_config=model_config, kv_transfer_manager=manager)
|
||||
|
||||
expected = ["req_0_0_1_1"]
|
||||
self.assertEqual(host.get_rank_aware_kv_send_keys("req", from_stage=0, to_stage=1), expected)
|
||||
self.assertEqual(host.get_rank_aware_kv_keys("req", from_stage=0, to_stage=1), expected)
|
||||
self.assertEqual(host.get_kv_rank_mapping()["from_tp"], 2)
|
||||
self.assertEqual(host.get_kv_rank_mapping()["to_tp"], 2)
|
||||
host.shutdown_omni_connectors()
|
||||
|
||||
def test_send_keys_route_from_rank_gt_to_rank(self):
|
||||
host = self._make_host(from_tp=4, to_tp=2, local_rank=3)
|
||||
self.assertEqual(host.get_rank_aware_kv_send_keys("req", from_stage=0), ["req_0_0_3_1"])
|
||||
|
||||
Reference in new issue
Block a user