Merge main and preserve dispatch backlog across monitor windows

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-09-21 17:13:27 +08:00
commit ea7de7e3de
769 files changed
+73802 -22805

No files matched your search

+10
View File
@@ -3,6 +3,7 @@
import pytest
import torch
from vllm.platforms import current_platform
from vllm_omni.attention import fish_kvcache_attn, fish_kvcache_backend
@@ -113,6 +114,15 @@ def test_fish_kvcache_enabled_by_default(monkeypatch):
assert not fish_kvcache_attn.is_fish_kvcache_attn_required()
def test_fish_kvcache_fast_path_is_unavailable_off_cuda(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
monkeypatch.setattr(
fish_kvcache_attn, "_triton_backend", lambda: pytest.fail("Triton backend must not load off CUDA")
)
assert not fish_kvcache_attn.is_available()
@pytest.mark.parametrize("value", ["", "1", "true", "yes", "on", "required"])
def test_fish_kvcache_enabled_values_are_consistent(monkeypatch, value):
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", value)
@@ -1,15 +1,289 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for omni_duplex_eval_media — PyAV-backed media helpers."""
from __future__ import annotations
import io
from pathlib import Path
import av
import numpy as np
import pytest
from vllm_omni.benchmarks.duplex.omni_duplex_eval_media import materialize_media
from vllm_omni.benchmarks.duplex.omni_duplex_eval_media import (
_ffmpeg_q_to_pil_quality,
extract_jpeg,
materialize_media,
read_audio_pcm16,
video_duration,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# ---------------------------------------------------------------------------
# Helpers: synthetic test media generated with PyAV (no external files)
# ---------------------------------------------------------------------------
def _make_synthetic_mp4(duration: float = 1.0, fps: int = 30) -> bytes:
"""Return an MP4 byte blob with a solid-color pattern.
Uses the built-in ``mpeg4`` encoder — available everywhere PyAV's bundled
FFmpeg is present — so no external codec dependency.
"""
buf = io.BytesIO()
total_frames = int(duration * fps)
with av.open(buf, mode="w", format="mp4") as container:
stream = container.add_stream("mpeg4", rate=fps)
stream.width = 64
stream.height = 64
stream.pix_fmt = "yuv420p"
for i in range(total_frames):
frame = av.VideoFrame.from_ndarray(np.full((64, 64, 3), (i * 8) % 256, dtype=np.uint8), format="rgb24")
for pkt in stream.encode(frame):
container.mux(pkt)
for pkt in stream.encode():
container.mux(pkt)
return buf.getvalue()
def _make_synthetic_wav(duration: float = 1.0, sample_rate: int = 16000) -> bytes:
"""Return a 16-bit mono PCM WAV byte blob (440 Hz sine wave)."""
import wave as _wave
n = int(sample_rate * duration)
samples = (np.sin(2 * np.pi * 440 * np.arange(n) / sample_rate) * 32767).astype(np.int16)
buf = io.BytesIO()
with _wave.open(buf, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2)
wf.setframerate(sample_rate)
wf.writeframes(samples.tobytes())
return buf.getvalue()
def _make_synthetic_ogg(duration: float = 1.0, sample_rate: int = 16000) -> bytes:
"""Return an OGG Vorbis byte blob (440 Hz sine wave).
Uses ``libopus`` as a fallback if ``libvorbis`` is unavailable in the
bundled FFmpeg (e.g. PyAV wheels from an older build).
"""
buf = io.BytesIO()
n = int(sample_rate * duration)
samples = (np.sin(2 * np.pi * 440 * np.arange(n) / sample_rate) * 0.3).astype(np.float32)
with av.open(buf, mode="w", format="ogg") as container:
codec_name = "libopus" if "libvorbis" not in av.codecs_available else "libvorbis"
stream = container.add_stream(codec_name, rate=sample_rate, layout="mono")
frame = av.AudioFrame.from_ndarray(samples[np.newaxis, :], format="fltp", layout="mono")
frame.sample_rate = sample_rate
for pkt in stream.encode(frame):
container.mux(pkt)
for pkt in stream.encode():
container.mux(pkt)
return buf.getvalue()
# ---------------------------------------------------------------------------
# video_duration
# ---------------------------------------------------------------------------
class TestVideoDuration:
@staticmethod
def test_returns_duration(tmp_path: Path) -> None:
mp4 = _make_synthetic_mp4(duration=2.0, fps=30)
path = tmp_path / "test.mp4"
path.write_bytes(mp4)
dur = video_duration(path)
assert abs(dur - 2.0) < 0.05, f"expected ~2.0s, got {dur}"
@staticmethod
def test_raises_for_missing_file(tmp_path: Path) -> None:
with pytest.raises((OSError, ValueError)):
video_duration(tmp_path / "nonexistent.mp4")
@staticmethod
def test_raises_for_broken_file(tmp_path: Path) -> None:
broken = tmp_path / "broken.mp4"
broken.write_bytes(b"\x00\x00\x00\x00")
with pytest.raises((OSError, ValueError)):
video_duration(broken)
# ---------------------------------------------------------------------------
# extract_jpeg
# ---------------------------------------------------------------------------
class TestExtractJpeg:
@staticmethod
def test_returns_valid_jpeg(tmp_path: Path) -> None:
mp4 = _make_synthetic_mp4(duration=1.0, fps=30)
path = tmp_path / "test.mp4"
path.write_bytes(mp4)
jpeg = extract_jpeg(path, timestamp=0.5, quality=3)
assert jpeg.startswith(b"\xff\xd8"), "not a valid JPEG header"
assert len(jpeg) > 100
@staticmethod
def test_quality_maps_to_pil_scale(tmp_path: Path) -> None:
"""Higher ffmpeg quality (lower q value) produces a larger JPEG."""
mp4 = _make_synthetic_mp4(duration=1.0, fps=30)
path = tmp_path / "test.mp4"
path.write_bytes(mp4)
high = extract_jpeg(path, timestamp=0.5, quality=1)
low = extract_jpeg(path, timestamp=0.5, quality=31)
assert len(high) > len(low), "quality=1 should produce larger JPEG than quality=31"
@staticmethod
def test_negative_timestamp_clamped(tmp_path: Path) -> None:
"""Negative timestamps are clamped to 0."""
mp4 = _make_synthetic_mp4(duration=1.0, fps=30)
path = tmp_path / "test.mp4"
path.write_bytes(mp4)
jpeg = extract_jpeg(path, timestamp=-1.0, quality=3)
assert jpeg.startswith(b"\xff\xd8")
@staticmethod
def test_raises_value_error_for_no_video(tmp_path: Path) -> None:
wav = _make_synthetic_wav()
path = tmp_path / "test.wav"
path.write_bytes(wav)
with pytest.raises(ValueError):
extract_jpeg(path, timestamp=0.0)
@staticmethod
def test_long_gop_returns_different_frames_for_different_timestamps(tmp_path: Path) -> None:
"""Long GOP: different timestamps must return different frame content (P2)."""
# Create a long-GOP video: 3s @ 10fps = 30 frames, gop_size=30 (1 keyframe)
buf = io.BytesIO()
fps = 10
duration = 3.0
total_frames = int(duration * fps)
with av.open(buf, mode="w", format="mp4") as container:
stream = container.add_stream("mpeg4", rate=fps)
stream.width = 64
stream.height = 64
stream.pix_fmt = "yuv420p"
stream.gop_size = 30 # 3 seconds per keyframe
for i in range(total_frames):
frame = av.VideoFrame.from_ndarray(np.full((64, 64, 3), (i * 64) % 256, dtype=np.uint8), format="rgb24")
for pkt in stream.encode(frame):
container.mux(pkt)
for pkt in stream.encode():
container.mux(pkt)
mp4 = buf.getvalue()
path = tmp_path / "long_gop.mp4"
path.write_bytes(mp4)
# Extract frames at different timestamps
frame0 = extract_jpeg(path, timestamp=0.0, quality=3)
frame1 = extract_jpeg(path, timestamp=1.0, quality=3)
frame2 = extract_jpeg(path, timestamp=2.0, quality=3)
# Verify all are valid JPEGs
assert frame0.startswith(b"\xff\xd8")
assert frame1.startswith(b"\xff\xd8")
assert frame2.startswith(b"\xff\xd8")
# Verify content differs: decode JPEGs and compare mean brightness
from PIL import Image
img0 = np.asarray(Image.open(io.BytesIO(frame0)))
img1 = np.asarray(Image.open(io.BytesIO(frame1)))
img2 = np.asarray(Image.open(io.BytesIO(frame2)))
means = [img.mean() for img in (img0, img1, img2)]
assert len(set(round(m, 1) for m in means)) >= 2, (
f"timestamps 0/1/2s should give different frames, got means {means}"
)
# ---------------------------------------------------------------------------
# read_audio_pcm16
# ---------------------------------------------------------------------------
class TestReadAudioPcm16:
@staticmethod
def test_wav_fast_path(tmp_path: Path) -> None:
"""WAV with matching params uses the fast wave module path."""
wav = _make_synthetic_wav(duration=1.0, sample_rate=16000)
path = tmp_path / "test.wav"
path.write_bytes(wav)
pcm = read_audio_pcm16(path)
assert len(pcm) == 16000 * 2, f"expected 32000 bytes, got {len(pcm)}"
assert isinstance(pcm, bytes)
@staticmethod
def test_wav_different_rate_resamples(tmp_path: Path) -> None:
"""A 48 kHz WAV is resampled down to 16 kHz — exact sample count after flush."""
wav = _make_synthetic_wav(duration=1.0, sample_rate=48000)
path = tmp_path / "test.wav"
path.write_bytes(wav)
pcm = read_audio_pcm16(path)
assert len(pcm) > 0
assert len(pcm) % 2 == 0
# 1s @ 48kHz → 1s @ 16kHz = 16000 samples = 32000 bytes (±1 sample)
assert abs(len(pcm) - 32000) <= 4, f"expected ~32000 bytes, got {len(pcm)}"
@staticmethod
def test_non_wav_ogg(tmp_path: Path) -> None:
"""Non-WAV audio (OGG) is decoded and resampled via PyAV."""
ogg = _make_synthetic_ogg(duration=1.0, sample_rate=16000)
path = tmp_path / "test.ogg"
path.write_bytes(ogg)
pcm = read_audio_pcm16(path)
assert len(pcm) > 0
assert len(pcm) % 2 == 0
@staticmethod
def test_non_wav_ogg_resample_accuracy(tmp_path: Path) -> None:
"""OGG resample should produce exact sample count after flush."""
ogg = _make_synthetic_ogg(duration=1.5, sample_rate=48000)
path = tmp_path / "test.ogg"
path.write_bytes(ogg)
pcm = read_audio_pcm16(path)
# 1.5s @ 48kHz → 1.5s @ 16kHz = 24000 samples = 48000 bytes (±1 sample)
expected = int(1.5 * 16000) * 2
assert abs(len(pcm) - expected) <= 4, f"expected ~{expected} bytes, got {len(pcm)}"
@staticmethod
def test_raises_for_broken_file(tmp_path: Path) -> None:
"""Corrupt audio must raise, not return empty bytes (L1)."""
broken = tmp_path / "broken.mp3"
broken.write_bytes(b"\x00\x00\x00\x00")
with pytest.raises((OSError, ValueError)):
read_audio_pcm16(broken)
# ---------------------------------------------------------------------------
# _ffmpeg_q_to_pil_quality
# ---------------------------------------------------------------------------
class TestFfmpegQToPilQuality:
@staticmethod
def test_q1_maps_to_best() -> None:
assert _ffmpeg_q_to_pil_quality(1) >= 90
@staticmethod
def test_q31_maps_to_lowest() -> None:
assert _ffmpeg_q_to_pil_quality(31) <= 10
@staticmethod
def test_monotonic() -> None:
"""Higher ffmpeg q → lower PIL quality (monotonically non-increasing)."""
prev = 100
for q in range(1, 32):
p = _ffmpeg_q_to_pil_quality(q)
assert p <= prev, f"quality.increase at q={q}: {prev} → {p}"
prev = p
def test_dict_with_bytes_materializes_a_real_file(tmp_path):
# A Hugging Face Audio/Video feature value {"bytes": ..., "path": ...} used
# to short-circuit on ``path`` (the artifact name inside the dataset, which
@@ -0,0 +1,74 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Verify metric sample counts survive benchmark JSON serialization."""
import asyncio
import json
import time
import pytest
from vllm.benchmarks.datasets import SampleRequest
from vllm.benchmarks.serve import TaskType
from vllm_omni.benchmarks.patch import patch
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
@pytest.mark.parametrize("selected_metrics", [["ttft", "tpot", "itl"], []])
@pytest.mark.parametrize("measured", [True, False])
def test_benchmark_result_preserves_tpot_sample_count(monkeypatch, measured, selected_metrics):
async def request_func(**kwargs):
return patch.MixRequestFuncOutput(
success=True,
prompt_len=2,
output_tokens=3,
generated_text="hello world!",
start_time=time.perf_counter(),
ttft=0.01,
latency=0.03,
text_latency=0.03,
itl=[0.01, 0.01] if measured else [0.0, 0.0],
)
monkeypatch.setitem(patch.ASYNC_REQUEST_FUNCS, "test-result", request_func)
result = asyncio.run(
patch.benchmark(
task_type=TaskType.GENERATION,
endpoint_type="test-result",
api_url="http://unused/v1/chat/completions",
base_url="http://unused",
model_id="test-model",
model_name="test-model",
tokenizer=None,
input_requests=[SampleRequest(prompt="hi", prompt_len=2, expected_output_len=3)],
logprobs=None,
request_rate=float("inf"),
burstiness=1.0,
disable_tqdm=True,
num_warmups=0,
profile=False,
selected_percentile_metrics=selected_metrics,
selected_percentiles=[50.0],
ignore_eos=False,
goodput_config_dict={},
max_concurrency=1,
lora_modules=None,
extra_headers=None,
extra_body=None,
ready_check_timeout_sec=0,
)
)
# DFX reads this dictionary from the benchmark's JSON result file.
saved_result = json.loads(json.dumps(result))
assert saved_result["completed"] == 1
assert saved_result.get("num_tpot_samples") == int(measured)
assert saved_result["num_ttft_samples"] == 1
assert saved_result["num_itl_samples"] == 2
assert "num_audio_ttfp_samples" in saved_result
assert "num_audio_rtf_samples" in saved_result
if measured and "tpot" in selected_metrics:
assert saved_result["mean_tpot_ms"] == pytest.approx(10.0)
else:
assert "mean_tpot_ms" not in saved_result
@@ -47,11 +47,50 @@ from benchmarks.accuracy.text_to_image.gbench import (
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
build_serve_common_argv,
seed_tts_bench_argv,
videomme_bench_argv,
)
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
def test_videomme_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", "lmms-eval/Video-MME")
monkeypatch.delenv("VLLM_VIDEOMME_REPO", raising=False)
monkeypatch.delenv("VIDEOMME_ROOT", raising=False)
argv = videomme_bench_argv()
dataset_idx = argv.index("--dataset-path")
assert argv[argv.index("--dataset-name") + 1] == "videomme"
assert argv[dataset_idx + 1] == "lmms-eval/Video-MME"
def test_videomme_bench_argv_uses_local_directory(monkeypatch, tmp_path: Path):
monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", str(tmp_path))
monkeypatch.delenv("VIDEOMME_ROOT", raising=False)
argv = videomme_bench_argv()
assert argv[argv.index("--dataset-path") + 1] == str(tmp_path.resolve())
def test_sync_dataset_env_preserves_videomme_hf_repo_id(monkeypatch):
ns = argparse.Namespace(
daily_omni_repo=None,
daily_omni_qa_json=None,
daily_omni_video_dir=None,
videomme_repo=None,
videomme_dataset_path="lmms-eval/Video-MME",
seed_tts_dataset_path=None,
seed_tts_root=None,
)
monkeypatch.delenv("VLLM_VIDEOMME_DATASET_PATH", raising=False)
sync_dataset_env_from_ns(ns)
assert os.environ["VLLM_VIDEOMME_DATASET_PATH"] == "lmms-eval/Video-MME"
def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
monkeypatch.setenv("VLLM_SEED_TTS_DATASET_PATH", "zhaochenyang20/seed-tts-eval")
monkeypatch.delenv("VLLM_SEED_TTS_REPO", raising=False)
@@ -0,0 +1,128 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import importlib
import sys
import pytest
from benchmarks.diffusion import backends
from benchmarks.diffusion.backends import RequestFuncInput, RequestFuncOutput
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu, pytest.mark.asyncio]
ENDPOINT = "/v1/chat/completions"
TEST_MODEL = "test-model"
@pytest.fixture
def diffusion_benchmark(monkeypatch):
# The standalone script imports its sibling backend module by name.
monkeypatch.setitem(sys.modules, "backends", backends)
return importlib.import_module("benchmarks.diffusion.diffusion_benchmark_serving")
@pytest.fixture
def warmup_args(diffusion_benchmark):
from benchmarks.diffusion.diffusion_benchmark_serving import build_parser
parser = build_parser()
return parser.parse_args(
[
f"--endpoint={ENDPOINT}",
"--dataset=random",
"--task=t2i",
"--warmup-requests=2",
"--warmup-num-inference-steps=4",
]
)
@pytest.fixture
def input_requests():
return [
RequestFuncInput(
prompt=prompt,
api_url=f"http://test.local{ENDPOINT}",
model=TEST_MODEL,
num_inference_steps=20,
)
for prompt in ("first", "second")
]
async def test_disabled_warmup_sends_no_requests(diffusion_benchmark, warmup_args, input_requests):
warmup_args.warmup_requests = 0
async def unexpected_request(*args):
pytest.fail("disabled warmup must not send requests")
pairs = await diffusion_benchmark._run_warmups(input_requests, warmup_args, None, unexpected_request)
assert pairs == []
async def test_successful_warmups_preserve_request_output_pairs(diffusion_benchmark, warmup_args, input_requests):
outputs = [RequestFuncOutput(success=True, latency=0.5), RequestFuncOutput(success=True, latency=1.0)]
pending_outputs = iter(outputs)
async def successful_request(req, session, pbar):
return next(pending_outputs)
pairs = await diffusion_benchmark._run_warmups(input_requests, warmup_args, None, successful_request)
assert [request.prompt for request, _ in pairs] == ["first", "second"]
assert [request.num_inference_steps for request, _ in pairs] == [4, 4]
assert [output for _, output in pairs] == outputs
assert [request.num_inference_steps for request in input_requests] == [20, 20]
@pytest.mark.parametrize("failed_count", [1, 2], ids=["partial-failure", "all-failed"])
@pytest.mark.parametrize(
"error",
["HTTP 503: private prompt rejected", "Cannot connect to https://test.local/?token=private"],
ids=["http-error", "transport-error"],
)
async def test_failed_warmups_abort_without_exposing_response_errors(
diffusion_benchmark, warmup_args, input_requests, failed_count, error
):
outputs = iter(
[RequestFuncOutput(success=False, error=error) for _ in range(failed_count)]
+ [RequestFuncOutput(success=True) for _ in range(2 - failed_count)]
)
async def request(req, session, pbar):
return next(outputs)
with pytest.raises(RuntimeError, match=f"{failed_count}/2 warmup requests failed") as exc_info:
await diffusion_benchmark._run_warmups(input_requests, warmup_args, None, request)
assert error not in str(exc_info.value)
assert "server logs" in str(exc_info.value)
async def test_benchmark_does_not_start_measurement_after_failed_warmup(
monkeypatch, mocker, diffusion_benchmark, warmup_args, input_requests
):
sent = []
async def failed_request(req, session, pbar):
sent.append(req)
return RequestFuncOutput(success=False, error="HTTP 503: unavailable")
monkeypatch.setitem(
diffusion_benchmark.backends_function_mapping["2i"],
warmup_args.endpoint,
(failed_request, warmup_args.endpoint),
)
dataset = mocker.Mock(spec=diffusion_benchmark.RandomDataset)
dataset.get_requests.return_value = input_requests
monkeypatch.setattr(diffusion_benchmark, "RandomDataset", mocker.Mock(return_value=dataset))
timer = mocker.Mock(spec=diffusion_benchmark.time)
timer.perf_counter.side_effect = AssertionError("measurement started after failed warmup")
monkeypatch.setattr(diffusion_benchmark, "time", timer)
with pytest.raises(RuntimeError, match="2/2 warmup requests failed"):
await diffusion_benchmark.benchmark(warmup_args)
assert len(sent) == warmup_args.warmup_requests
@@ -0,0 +1,546 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Offline Video-MME sources: local mirrors must not fall back to a Hub id."""
from __future__ import annotations
import os
import zipfile
from argparse import Namespace
from pathlib import Path
from typing import Any
from urllib.parse import unquote, urlparse
import pytest
from PIL import Image
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import _validate_videomme
from vllm_omni.benchmarks.data_modules.videomme_dataset import (
VideoMMEDataset,
VideoMMESampleRequest,
cached_videomme_snapshots,
ensure_videomme_hub_root,
ensure_videomme_videos_extracted,
resolve_videomme_local_root,
videomme_local_parquet,
videomme_local_subtitle_dir,
videomme_local_video_dir,
)
from vllm_omni.benchmarks.data_modules.videomme_eval import (
compute_videomme_accuracy_metrics,
extract_characters_regex,
normalize_gold_answer,
)
from vllm_omni.benchmarks.patch.patch import _looks_like_hf_dataset_id, _videomme_repo_from_args, get_samples
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
def _write_parquet_placeholder(root: Path) -> Path:
path = root / "videomme" / "test-00000-of-00001.parquet"
path.parent.mkdir(parents=True)
path.write_bytes(b"parquet")
return path
def test_resolve_local_root_returns_none_for_hub_id() -> None:
assert resolve_videomme_local_root("lmms-eval/Video-MME") is None
assert resolve_videomme_local_root("") is None
assert resolve_videomme_local_root(None) is None
def test_plain_local_mirror(tmp_path: Path) -> None:
pq = _write_parquet_placeholder(tmp_path)
video = tmp_path / "video" / "abc.mp4"
video.parent.mkdir()
video.write_bytes(b"fake")
(tmp_path / "subtitle").mkdir()
(tmp_path / "subtitle" / "abc.srt").write_text("1\n", encoding="utf-8")
root = resolve_videomme_local_root(str(tmp_path))
assert root == tmp_path.resolve()
assert videomme_local_parquet(root) == pq.resolve()
assert videomme_local_video_dir(root) == tmp_path.resolve() / "video"
assert videomme_local_subtitle_dir(root) == tmp_path.resolve() / "subtitle"
def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
cache_dir = tmp_path / "datasets--lmms-eval--Video-MME"
snapshot = cache_dir / "snapshots" / "deadbeef"
snapshot.mkdir(parents=True)
_write_parquet_placeholder(snapshot)
(cache_dir / "refs").mkdir()
(cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
root = resolve_videomme_local_root(str(cache_dir))
assert root == snapshot.resolve()
assert videomme_local_parquet(root) is not None
def test_hf_cache_dir_with_unusable_ref_falls_back_to_revision(tmp_path: Path) -> None:
cache_dir = tmp_path / "datasets--lmms-eval--Video-MME"
snapshot = cache_dir / "snapshots" / "deadbeef"
snapshot.mkdir(parents=True)
_write_parquet_placeholder(snapshot)
(cache_dir / "refs").mkdir()
# An empty ref must not resolve to snapshots/, which holds no dataset files.
(cache_dir / "refs" / "main").write_text("", encoding="utf-8")
assert resolve_videomme_local_root(str(cache_dir)) == snapshot.resolve()
def _stub_failing_hub(monkeypatch: pytest.MonkeyPatch, hub_cache: Path) -> dict[str, object]:
"""Point the Hub cache at ``hub_cache`` and make every snapshot_download miss."""
from huggingface_hub import constants as hf_constants
from huggingface_hub.errors import LocalEntryNotFoundError
from vllm_omni.transformers_utils import repo_utils
seen: dict[str, object] = {}
class _Api:
def snapshot_download(self, **kwargs: object) -> str:
seen.update(kwargs)
raise LocalEntryNotFoundError("no cached snapshot")
monkeypatch.setattr(hf_constants, "HF_HUB_OFFLINE", True)
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(hub_cache))
monkeypatch.setattr(repo_utils, "hf_api", lambda: _Api())
return seen
def test_hub_root_offline_without_cache_points_at_local_options(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
seen = _stub_failing_hub(monkeypatch, tmp_path)
with pytest.raises(FileNotFoundError, match="--videomme-video-dir") as excinfo:
ensure_videomme_hub_root("lmms-lab/Video-MME")
# Offline runs must consult the cache directly instead of resolving over the network.
assert seen["local_files_only"] is True
assert "already cached under another repo id" not in str(excinfo.value)
def test_hub_root_offline_names_snapshot_cached_under_another_repo_id(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
sibling = tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef"
sibling.mkdir(parents=True)
_write_parquet_placeholder(sibling)
_stub_failing_hub(monkeypatch, tmp_path)
with pytest.raises(FileNotFoundError) as excinfo:
ensure_videomme_hub_root("lmms-lab/Video-MME")
assert str(sibling.resolve()) in str(excinfo.value)
def test_cached_videomme_snapshots_skips_partial_download(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
from huggingface_hub import constants as hf_constants
(tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef").mkdir(parents=True)
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
assert cached_videomme_snapshots() == []
def test_extract_videos_rejects_empty_tree(tmp_path: Path) -> None:
(tmp_path / "video").mkdir()
with pytest.raises(FileNotFoundError):
ensure_videomme_videos_extracted(tmp_path)
assert not (tmp_path / ".videomme_videos_extracted").exists()
def test_nested_unzipped_videos_are_discovered(tmp_path: Path) -> None:
nested = tmp_path / "videos" / "videos_chunked_01" / "data" / "xyz.mp4"
nested.parent.mkdir(parents=True)
nested.write_bytes(b"fake")
assert videomme_local_video_dir(tmp_path) == tmp_path / "videos"
def test_extract_characters_regex_matches_official_prefixes() -> None:
assert extract_characters_regex("The best answer is B.") == "B"
assert extract_characters_regex("Answer: C") == "C"
assert extract_characters_regex("I think D is correct") == "D"
assert extract_characters_regex("no letter here at all in this long sentence") is None
def test_normalize_gold_answer() -> None:
assert normalize_gold_answer("a") == "A"
assert normalize_gold_answer("B. content") == "B"
assert normalize_gold_answer("") is None
class _Out:
def __init__(self, text: str, *, success: bool = True) -> None:
self.generated_text = text
self.success = success
self.error = "" if success else "http fail"
def test_compute_videomme_accuracy_metrics() -> None:
reqs = [
VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="A",
videomme_video_id="v1",
videomme_question_id="q1",
videomme_duration="short",
videomme_domain="Knowledge",
videomme_sub_category="Science",
videomme_task_type="QA",
),
VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="B",
videomme_video_id="v2",
videomme_question_id="q2",
videomme_duration="long",
videomme_domain="Knowledge",
videomme_sub_category="Science",
videomme_task_type="QA",
),
]
metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("The answer is C")])
assert metrics is not None
assert metrics["videomme_correct"] == 1
assert metrics["videomme_evaluated_ok"] == 2
assert metrics["videomme_accuracy"] == 0.5
assert metrics["videomme_submitted"] == 2
assert metrics["videomme_unique_question_ids"] == 2
assert _validate_videomme(metrics, min_accuracy=0.4) == []
assert "videomme_eval_items" not in metrics
assert metrics["videomme_per_duration_accuracy"]["short"] == 1.0
assert metrics["videomme_per_duration_accuracy"]["long"] == 0.0
def test_compute_videomme_accuracy_metrics_http_fail_excluded_from_default() -> None:
reqs = [
VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="A",
videomme_question_id="q1",
),
VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="B",
videomme_question_id="q2",
),
]
metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("", success=False)])
assert metrics is not None
assert metrics["videomme_accuracy"] == 1.0
assert metrics["videomme_accuracy_incl_http_fail"] == 0.5
assert metrics["videomme_request_failed"] == 1
errs = _validate_videomme(metrics, min_accuracy=0.68)
assert any("videomme_request_failed" in err for err in errs)
def test_compute_videomme_accuracy_metrics_saves_eval_items() -> None:
req = VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="A",
videomme_question_id="q1",
)
metrics = compute_videomme_accuracy_metrics([req], [_Out("A")], include_per_item=True)
assert metrics is not None
assert metrics["videomme_eval_items"][0]["question_id"] == "q1"
def test_videomme_save_eval_items_cli_sets_env(monkeypatch: pytest.MonkeyPatch) -> None:
from vllm_omni.benchmarks import serve
seen: dict[str, str | None] = {}
async def fake_main_async(args: Namespace) -> dict[str, str]:
seen["env"] = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS")
return {"ok": "1"}
monkeypatch.delenv("VIDEOMME_SAVE_EVAL_ITEMS", raising=False)
monkeypatch.setattr(serve, "main_async", fake_main_async)
args = Namespace(
videomme_save_eval_items=True,
seed_tts_wer_eval=False,
seed_tts_wer_save_items=False,
daily_omni_save_eval_items=False,
omni_request_timeout_s=None,
endpoint=None,
backend="openai-chat-omni",
explicit_keys=frozenset(),
extra_body=None,
print_stage=False,
dataset_name="videomme",
)
assert serve.main(args) == {"ok": "1"}
assert seen["env"] == "1"
def _file_uri_path(url: str) -> Path:
parsed = urlparse(url)
assert parsed.scheme == "file"
return Path(unquote(parsed.path))
def test_relative_video_dir_emits_absolute_file_uris_cold_and_warm(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
videos = tmp_path / "videos"
videos.mkdir()
(videos / "vid1.mp4").write_bytes(b"fake")
def fake_extract(cls, video_path, *, include_audio, max_num_frames): # noqa: ANN001
return [Image.new("RGB", (4, 4), color="red")], []
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None)
monkeypatch.setattr(VideoMMEDataset, "_extract_frames_and_audio", classmethod(fake_extract))
ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1)
assert ds.video_dir == videos.resolve()
cold = ds._get_minicpm_frame_parts("vid1", include_audio=False)
assert cold is not None
cold_path = _file_uri_path(cold[0]["image_url"]["url"])
assert cold_path.is_absolute()
assert cold_path.is_file()
warm_ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1)
warm = warm_ds._get_minicpm_frame_parts("vid1", include_audio=False)
assert warm is not None
warm_path = _file_uri_path(warm[0]["image_url"]["url"])
assert warm_path.is_absolute()
assert warm_path == cold_path
def test_videomme_repo_from_args_honors_custom_hub_id() -> None:
args = Namespace(dataset_name="videomme", dataset_path="my-org/custom-videomme", hf_name=None)
assert _videomme_repo_from_args(args, explicit=True) == "my-org/custom-videomme"
def test_videomme_repo_from_args_defaults_when_path_omitted() -> None:
args = Namespace(dataset_name="videomme", dataset_path=None, hf_name=None)
assert _videomme_repo_from_args(args, explicit=True) is None
def test_videomme_repo_from_args_rejects_unsupported_path() -> None:
args = Namespace(dataset_name="videomme", dataset_path="not-a-local-dir-or-hub-id", hf_name=None)
with pytest.raises(ValueError, match="Unsupported Video-MME"):
_videomme_repo_from_args(args, explicit=True)
def test_get_samples_rejects_unsupported_videomme_dataset_path() -> None:
args = Namespace(
dataset_name="videomme",
dataset_path="not-a-local-dir-or-hub-id",
hf_name=None,
backend="openai-chat-omni",
seed=0,
)
with pytest.raises(ValueError, match="Unsupported Video-MME"):
get_samples(args, None)
def test_hf_dataset_name_does_not_auto_detect_custom_repo() -> None:
args = Namespace(dataset_name="hf", dataset_path="my-org/custom-videomme", hf_name=None)
assert _videomme_repo_from_args(args) is None
assert _looks_like_hf_dataset_id("my-org/custom-videomme")
assert not _looks_like_hf_dataset_id("./videos")
official = Namespace(dataset_name="hf", dataset_path="lmms-eval/Video-MME", hf_name=None)
assert _videomme_repo_from_args(official) == "lmms-eval/Video-MME"
@pytest.mark.parametrize("max_frames", [0, -1])
def test_rejects_nonpositive_max_frames(max_frames: int, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None)
with pytest.raises(ValueError, match="max_frames must be positive"):
VideoMMEDataset(parquet_path="unused.parquet", max_frames=max_frames)
def test_frame_cache_rebuild_overwrites_incomplete_files(tmp_path: Path) -> None:
frame = tmp_path / "frame_0000.jpg"
frame.write_bytes(b"interrupted JPEG")
VideoMMEDataset._emit_media(tmp_path, frame.name, b"complete JPEG", "image_url", "image/jpeg")
assert frame.read_bytes() == b"complete JPEG"
def test_frame_cache_rejects_empty_media(tmp_path: Path) -> None:
(tmp_path / "manifest.json").write_text('{"count": 1}', encoding="utf-8")
(tmp_path / "frame_0000.jpg").touch()
dataset = object.__new__(VideoMMEDataset)
assert dataset._cached_parts_from_disk(tmp_path, include_audio=False) is None
@pytest.mark.parametrize("case", ["duplicates", "missing_gold"])
def test_accuracy_gate_rejects_incomplete_question_coverage(case: str) -> None:
def request(question_id: str, gold: str) -> VideoMMESampleRequest:
return VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_question_id=question_id,
videomme_gold_answer=gold,
)
requests = [
request("q1", "A"),
request("q1" if case == "duplicates" else "q2", "" if case == "missing_gold" else "A"),
]
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A"), _Out("A")])
assert metrics is not None
assert metrics["videomme_accuracy"] == 1.0
errors = _validate_videomme(metrics, min_accuracy=0.68)
assert any(("unique" if case == "duplicates" else "no_gold") in error for error in errors)
@pytest.fixture
def videomme_mirror(tmp_path: Path) -> Path:
import pandas as pd
parquet = tmp_path / "videomme" / "test-00000-of-00001.parquet"
parquet.parent.mkdir()
pd.DataFrame(
[
{
"videoID": f"v{i}",
"question_id": f"q{i}",
"question": "Which option is correct?",
"options": ["A. first", "B. second", "C. third", "D. fourth"],
"answer": "A",
"duration": duration,
}
for i, duration in enumerate(("short", "long"), start=1)
]
).to_parquet(parquet)
videos = tmp_path / "video"
videos.mkdir()
for i in (1, 2):
(videos / f"v{i}.mp4").write_bytes(b"video")
return tmp_path
def _sample_video_urls(dataset: VideoMMEDataset, monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> list:
monkeypatch.setattr(
"vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer
)
return dataset.sample(Namespace(encode=lambda text: [1]), **kwargs)
@pytest.mark.parametrize("no_oversample", [True, False])
def test_missing_video_cannot_pass_accuracy_gate(
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch, no_oversample: bool
) -> None:
(videomme_mirror / "video" / "v2.mp4").unlink()
dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", disable_shuffle=True)
requests = _sample_video_urls(dataset, monkeypatch, num_requests=2, no_oversample=no_oversample)
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests])
assert metrics is not None
assert metrics["videomme_accuracy"] == 1.0
assert metrics["videomme_submitted"] == (1 if no_oversample else 2)
assert metrics["videomme_skipped_rows"] == 1
assert any("videomme_skipped_rows" in error for error in _validate_videomme(metrics, min_accuracy=0.68))
def test_complete_duration_subset_can_pass_accuracy_gate(
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", duration_filter="short")
requests = _sample_video_urls(dataset, monkeypatch, num_requests=2700, no_oversample=True)
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests])
assert metrics is not None
assert metrics["videomme_submitted"] == 1
assert metrics["videomme_skipped_rows"] == 0
assert _validate_videomme(metrics, min_accuracy=0.68) == []
def test_dataset_direct_hub_loading_resolves_video_assets(
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
downloaded = []
def download(repo_id: str) -> Path:
downloaded.append(repo_id)
return videomme_mirror
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.ensure_videomme_hub_root", download)
dataset = VideoMMEDataset(dataset_path="my-org/custom-videomme", pack_mode="video_url")
assert downloaded == ["my-org/custom-videomme"]
assert dataset.video_dir == videomme_mirror / "video"
assert len(dataset.data) == 2
def test_local_source_with_explicit_parquet_keeps_video_root(videomme_mirror: Path) -> None:
dataset = VideoMMEDataset(
dataset_path=str(videomme_mirror),
parquet_path=str(videomme_local_parquet(videomme_mirror)),
pack_mode="video_url",
)
assert dataset.video_dir == videomme_mirror / "video"
assert len(dataset.data) == 2
def test_local_source_without_parquet_fails_locally(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def unexpected_hub(*args, **kwargs):
pytest.fail("An explicitly selected local mirror must not fall back to Hub loading")
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.load_dataset", unexpected_hub)
with pytest.raises(FileNotFoundError, match="No Video-MME parquet"):
VideoMMEDataset(dataset_path=str(tmp_path))
def test_get_samples_prefers_local_root_over_hf_name(videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch) -> None:
args = Namespace(
dataset_name="videomme",
dataset_path=str(videomme_mirror),
hf_name="unused-hf-name",
backend="openai-chat-omni",
seed=0,
videomme_pack_mode="video_url",
num_prompts=2,
request_id_prefix="",
no_oversample=True,
)
monkeypatch.setattr(
"vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer
)
requests = get_samples(args, Namespace(encode=lambda text: [1]))
assert len(requests) == 2
assert {request.videomme_question_id for request in requests} == {"q1", "q2"}
def test_extraction_replaces_truncated_video(tmp_path: Path) -> None:
video = tmp_path / "video" / "v1.mp4"
video.parent.mkdir()
video.write_bytes(b"partial")
with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive:
archive.writestr("nested/v1.mp4", b"complete video")
assert ensure_videomme_videos_extracted(tmp_path) == video.parent
assert video.read_bytes() == b"complete video"
def test_failed_extraction_does_not_publish_partial_video(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive:
archive.writestr("v1.mp4", b"complete video")
def interrupted_copy(src, dst):
dst.write(b"partial")
raise OSError("interrupted")
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.shutil.copyfileobj", interrupted_copy)
with pytest.raises(OSError, match="interrupted"):
ensure_videomme_videos_extracted(tmp_path)
assert list((tmp_path / "video").iterdir()) == []
assert not (tmp_path / ".videomme_videos_extracted").exists()
+24
View File
@@ -11,6 +11,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
AMD_MERGE_PIPELINE = Path(".buildkite/amd/test-amd-merge.yml")
AMD_NIGHTLY_PIPELINE = Path(".buildkite/amd/test-amd-nightly.yml")
AMD_READY_PIPELINE = Path(".buildkite/amd/test-amd-ready.yml")
AMD_TEMPLATE = Path(".buildkite/amd/test-template-amd-omni.j2")
def _find_step(label: str, pipeline_path: Path = AMD_MERGE_PIPELINE) -> dict:
@@ -53,3 +55,25 @@ def test_qwen3_accuracy_defers_artifact_path_expansion() -> None:
assert '"$${BUILDKITE_BUILD_CHECKOUT_PATH:?}"' in staging_command
assert '"$$artifact_dir"' in staging_command
assert step["artifact_paths"] == ["tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"]
def test_ready_diffusion_cpu_suite_is_sharded() -> None:
step = _find_step("Simple · Diffusion Test · Shard %N/%t", AMD_READY_PIPELINE)
pytest_command = next(command for command in step["commands"] if "pytest" in command)
assert step["parallelism"] == 4
assert step["timeout_in_minutes"] == 45
assert "--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT" in pytest_command
assert "--shard-id=$$BUILDKITE_PARALLEL_JOB" in pytest_command
def test_cosyvoice_gpu_abort_gets_one_fresh_job_retry() -> None:
step = _find_step("CosyVoice3-TTS E2E Test", AMD_READY_PIPELINE)
assert step["retry"] == {"automatic": [{"exit_status": 134, "limit": 1}]}
template = AMD_TEMPLATE.read_text(encoding="utf-8")
# Both grouped and top-level AMD steps must preserve an explicit retry
# policy when the source suite is rendered into the uploaded pipeline.
assert template.count("{% if step.retry %}") == 2
assert template.count("{% for retry_rule in step.retry.automatic %}") == 2
+56 -7
View File
@@ -9,8 +9,10 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
REPO_ROOT = Path(__file__).resolve().parents[2]
AMD_TEMPLATE = REPO_ROOT / ".buildkite/amd/test-template-amd-omni.j2"
AMD_BUILD_SCRIPT = REPO_ROOT / ".buildkite/amd/scripts/build-ci-image.sh"
CI_DOCKERFILE = REPO_ROOT / "docker/Dockerfile.ci"
ROCM_DOCKERFILE = REPO_ROOT / "docker/Dockerfile.rocm"
ROCM_DOCKERIGNORE = REPO_ROOT / "docker/Dockerfile.rocm.dockerignore"
def _docker_arg(path: Path, name: str) -> str:
@@ -42,16 +44,63 @@ def test_rocm_defaults_to_prebuilt_base_image() -> None:
assert _docker_arg(ROCM_DOCKERFILE, "USE_NIGHTLY_BUILD") == "0"
def test_amd_build_uses_rocm_dockerfile_defaults() -> None:
def test_amd_build_uses_registry_cache() -> None:
template = AMD_TEMPLATE.read_text(encoding="utf-8")
build_commands = [line.strip() for line in template.splitlines() if '"docker build ' in line]
script_command = '"bash .buildkite/amd/scripts/build-ci-image.sh"'
script = AMD_BUILD_SCRIPT.read_text(encoding="utf-8")
assert len(build_commands) == 1, f"expected one AMD image build command, found {len(build_commands)}"
build_command = build_commands[0]
assert "-f docker/Dockerfile.rocm" in build_command
assert template.count(script_command) == 1
assert "docker buildx build --push" in script
assert '--cache-from "type=registry,ref=${CACHE_REF}"' in script
assert '--cache-to "type=registry,ref=${CACHE_REF}' in script
assert "mode=max" in script
def test_amd_cache_key_tracks_dependency_inputs_and_architecture() -> None:
script = AMD_BUILD_SCRIPT.read_text(encoding="utf-8")
for cache_input in (
"docker/Dockerfile.rocm",
"${DOCKERFILE}.dockerignore",
"pyproject.toml",
"setup.py",
"requirements",
"tools/install_torchcodec_rocm.sh",
"ROCM_ARCH",
):
assert cache_input in script
def test_amd_build_uses_rocm_dockerfile_defaults() -> None:
build_script = AMD_BUILD_SCRIPT.read_text(encoding="utf-8")
assert 'DOCKERFILE="docker/Dockerfile.rocm"' in build_script
for arg_name in ("BASE_IMAGE", "USE_NIGHTLY_BUILD"):
assert f"--build-arg {arg_name}" not in build_command
assert f"--build-arg={arg_name}" not in build_command
assert f"--build-arg {arg_name}" not in build_script
assert f"--build-arg={arg_name}" not in build_script
def test_rocm_source_copy_preserves_dependency_cache() -> None:
dockerfile = ROCM_DOCKERFILE.read_text(encoding="utf-8")
lines = dockerfile.splitlines()
source_copy_index = _line_index(lines, "COPY --link . .")
test_stage_index = _line_index(lines, "FROM base AS test")
assert "FROM deps AS base" in lines[:source_copy_index]
assert any('-e ".[dev]"' in line for line in lines[:source_copy_index])
assert not any(
line.startswith(("RUN ", "COPY ", "ADD ")) for line in lines[source_copy_index + 1 : test_stage_index]
)
def test_rocm_build_context_excludes_git_history() -> None:
patterns = {
line.strip()
for line in ROCM_DOCKERIGNORE.read_text(encoding="utf-8").splitlines()
if line.strip() and not line.lstrip().startswith("#")
}
assert ".git" in patterns
def test_rocm_source_ref_tracks_ci_vllm_release() -> None:
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Integration tests: strategy overlay on the real merge_pipeline_deploy seam."""
from __future__ import annotations
@@ -20,6 +20,7 @@ from vllm_omni.config.composable_parallel import (
apply_strategy_specs,
)
from vllm_omni.config.config_factory import StageConfigFactory
from vllm_omni.config.omni_config import VllmOmniConfig
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
@@ -137,3 +138,31 @@ def test_cli_overrides_strategy_with_warning():
assert _resolved(stages, "talker").runtime.num_replicas == 3
# ...and the override was warned about, naming the conflicting field.
assert any("num_replicas" in m and "overrides the strategy-derived" in m for m in messages)
def test_typed_cli_overrides_strategy():
pipeline = OMNI_PIPELINES["qwen2_5_omni"]
config = VllmOmniConfig.from_pipeline_config(
pipeline,
user_deploy_config=load_deploy_config(_DEPLOY),
cli_overrides={"stage_1_num_replicas": 3},
strategy_specs={"talker": [_stage_replica(2, "round_robin")]},
)
talker = next(stage for stage in config.stage_configs if stage.model_stage == "talker")
assert talker.runtime_config.num_replicas == 3
assert config.orchestrator_config.omni_lb_policy == "round-robin"
assert config.strategy_omni_lb_policy == "round-robin"
def test_typed_tp_only_strategy_has_no_derived_lb_policy():
pipeline = OMNI_PIPELINES["qwen2_5_omni"]
config = VllmOmniConfig.from_pipeline_config(
pipeline,
user_deploy_config=load_deploy_config(_DEPLOY),
cli_overrides={"omni_lb_policy": "round-robin"},
strategy_specs={"thinker": [_tp(1)]},
)
assert config.orchestrator_config.omni_lb_policy == "round-robin"
assert config.strategy_omni_lb_policy is None
+69
View File
@@ -1429,6 +1429,9 @@ stages:
assert deploy.async_chunk is True
assert deploy.connectors is not None
assert deploy.platforms is not None
connector = deploy.connectors["connector_of_shared_memory"]
assert "async_chunk_batch_min_size" not in connector["extra"]
assert "async_chunk_batch_max_wait_ms" not in connector["extra"]
@pytest.mark.parametrize(
("hf_config", "model"),
@@ -2625,6 +2628,15 @@ class TestPlatformOverrides:
assert stages[1].yaml_engine_args["dtype"] == dtype
def test_moss_tts_rocm_disables_codec_cudagraph(self):
deploy_path = Path(get_deploy_config_path("moss_tts.yaml"))
base = load_deploy_config(deploy_path)
assert base.stages[1].enforce_eager is False
rocm = _apply_platform_overrides(base, platform="rocm")
assert rocm.stages[1].enforce_eager is True
def test_higgs_audio_v3_rocm_uses_triton_attention(self):
deploy_path = Path(get_deploy_config_path("higgs_multimodal_qwen3.yaml"))
@@ -2688,6 +2700,63 @@ class TestPlatformOverrides:
# Explicit null clears the inherited single-GPU 2 GiB CUDA cap.
assert replica_stages[1].yaml_engine_args.get("kv_cache_memory_bytes") is None
def test_fish_speech_npu_uses_ascend_kv_block_size(self):
deploy_path = Path(get_deploy_config_path("fish_qwen3_omni.yaml"))
deploy = _apply_platform_overrides(load_deploy_config(deploy_path), platform="npu")
assert deploy.stages[0].engine_extras["block_size"] == 128
@pytest.mark.parametrize("platform", ["cuda", "npu"])
def test_recommended_native_runner_platform_defaults(self, platform):
filename, pipeline_key = "qwen3_tts_mrv2.yaml", "qwen3_tts"
deploy = load_deploy_config(Path(get_deploy_config_path(filename)))
deploy = _apply_platform_overrides(deploy, platform=platform)
expect_v2 = platform == "cuda"
assert deploy.model_runner == ("v2" if expect_v2 else "v1")
# The selection must reach the live worker-dispatch consumer, not just
# the transport-level DeployConfig field.
pipeline = resolve_pipeline_config(pipeline_key)
stages = merge_pipeline_deploy(pipeline, deploy)
assert [stage.yaml_engine_args["use_v2_model_runner"] for stage in stages] == [expect_v2] * len(stages)
if expect_v2 and pipeline.stages and deploy.async_chunk:
# v2 only engages the native plane on stages declaring support.
assert all(ps.supports_native_mrv2_data_plane for ps in pipeline.stages)
@pytest.mark.parametrize("runner,native", [("v1", False), ("v2", False), ("v2", True)])
def test_mrv2_undeclared_transport_warns(self, monkeypatch, runner, native):
from unittest.mock import Mock
warning = Mock()
monkeypatch.setattr("vllm_omni.config.stage_config.logger.warning", warning)
pipeline = PipelineConfig(
model_type="custom_pipeline",
stages=[
StagePipelineConfig(
stage_id=0, model_stage="custom_ar", final_output=True, supports_native_mrv2_data_plane=native
)
],
)
merge_pipeline_deploy(pipeline, DeployConfig(model_runner=runner))
assert warning.call_count == int(runner == "v2" and not native)
if warning.called:
assert "legacy transport path" in warning.call_args.args[0]
def test_runner_selection_rejects_engine_extras_override(self):
pipeline = PipelineConfig(
model_type="runner_selection_reserved",
stages=SINGLE_STAGE_PIPE_CFG,
)
for reserved in ("use_v2_model_runner", "supports_native_mrv2_data_plane"):
deploy = DeployConfig(
model_runner="v2",
stages=[StageDeployConfig(stage_id=0, engine_extras={reserved: False})],
)
with pytest.raises(ValueError, match=f"{reserved!r} must not be set"):
merge_pipeline_deploy(pipeline, deploy)
def test_npu_overrides(self):
deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml"))
if not deploy_path.exists():
+27 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for endpoint restrictions logic."""
import pytest
@@ -44,6 +44,32 @@ def test_restricted_completions_returns_400():
assert body["error"]["type"] == "BadRequestError"
def test_restricted_image_edits_returns_400():
"""Ensure pipelines can reject image edits independently of generation."""
app = FastAPI()
@app.post("/v1/images/edits")
async def existing_handler():
return {"ok": True}
@app.post("/v1/images/generations")
async def generations_handler():
return {"ok": True}
restrictions = (EndpointRestriction(OmniServingCapability.IMAGE_EDITS, REJECTION_REASON),)
shutdown_unsupported_routes(app, restrictions)
client = TestClient(app)
resp = client.post("/v1/images/edits")
assert resp.status_code == 400
assert resp.json()["error"]["message"] == REJECTION_REASON
# The restriction is endpoint-specific and must not remove generations.
resp = client.post("/v1/images/generations")
assert resp.status_code == 200
assert resp.json() == {"ok": True}
def test_unrestricted_completions_not_blocked():
"""Ensure that if we don't shutdown any routes, the route stays."""
app = _make_app_with_server_route()
+7 -7
View File
@@ -164,11 +164,11 @@ def test_inventory_matches_reviewed_snapshot_counts():
"""Make an inventory expansion an explicit review decision."""
category_counts = Counter(item.category for item in ENVIRONMENT_VARIABLE_INVENTORY.values())
assert category_counts == {
EnvironmentVariableCategory.PUBLIC_OMNI: 28,
EnvironmentVariableCategory.PUBLIC_OMNI: 29,
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
EnvironmentVariableCategory.MODEL_SPECIFIC: 64,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
EnvironmentVariableCategory.INTERNAL: 2,
}
@@ -178,11 +178,11 @@ def test_inventory_matches_reviewed_snapshot_counts():
if item.category is EnvironmentVariableCategory.MODEL_SPECIFIC
)
assert {disposition: disposition_counts[disposition] for disposition in ModelEnvironmentVariableDisposition} == {
ModelEnvironmentVariableDisposition.PROMOTE: 38,
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
ModelEnvironmentVariableDisposition.PROMOTE: 31,
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 5,
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
ModelEnvironmentVariableDisposition.INTERNALIZE: 15,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
ModelEnvironmentVariableDisposition.INTERNALIZE: 14,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 7,
}
@@ -1,11 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from vllm.pooling_params import PoolingParams
import vllm_omni.config.pipeline_registry # noqa: F401 (populate registry)
from vllm_omni.config.omni_config import VllmOmniConfig
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES as _PIPELINE_REGISTRY
from vllm_omni.config.stage_config import (
DeployConfig,
StageExecutionType,
)
from vllm_omni.engine.stage_init_utils import (
build_engine_args_dict_from_omni_stage_config,
extract_stage_metadata_from_omni_stage_config,
)
from vllm_omni.model_executor.stage_input_processors.forced_aligner import (
POOLING_OUTPUT_DECODER_PATH,
TIMESTAMPS_MODALITY,
@@ -69,3 +78,55 @@ def test_inject_noop_without_forced_aligner():
assert ext_pipeline is pipeline
assert len(ext_deploy.stages) == 0
@pytest.mark.parametrize("async_chunk", [True, False])
@pytest.mark.parametrize("cli_async_chunk", [None, True, False])
def test_aligner_uses_completed_audio_without_disabling_upstream_chunks(async_chunk, cli_async_chunk):
from vllm_omni.config.config_factory import StageConfigFactory
pipeline = _PIPELINE_REGISTRY["qwen3_tts"]
deploy = DeployConfig(async_chunk=async_chunk)
resolution = StageConfigFactory._resolve_legacy_from_registry(
pipeline,
{"forced_aligner": "/models/Qwen3-ForcedAligner-0.6B", "async_chunk": cli_async_chunk},
user_deploy_config=deploy,
)
stages = [stage.to_omegaconf() for stage in resolution.stage_configs]
expected_async = async_chunk if cli_async_chunk is None else cli_async_chunk
assert len(stages) == 3
assert [stage.engine_args.async_chunk for stage in stages] == [expected_async, expected_async, False]
assert stages[2].custom_process_input_func.endswith("code2wav2aligner")
assert stages[2].engine_args.runner == "pooling"
assert len(pipeline.stages) == 2
assert deploy.stages == []
def test_injected_aligner_survives_typed_config_and_runtime_projections():
pipeline, deploy = inject_forced_aligner_stage(
_PIPELINE_REGISTRY["qwen3_tts"],
DeployConfig(),
{"forced_aligner": "/models/Qwen3-ForcedAligner-0.6B"},
)
config = VllmOmniConfig.from_pipeline_config(
pipeline,
user_deploy_config=deploy,
cli_overrides={"model": "/models/Qwen3-TTS"},
)
stage = config.stage_configs[-1]
assert stage.pooling_config.runner == "pooling"
assert stage.pooling_config.pooling_output_decoder == POOLING_OUTPUT_DECODER_PATH
assert isinstance(stage.pooling_config.default_pooling_params, PoolingParams)
assert stage.pooling_config.default_pooling_params.task == deploy.stages[-1].default_pooling_params["task"]
metadata = extract_stage_metadata_from_omni_stage_config(stage)
assert isinstance(metadata.default_sampling_params, PoolingParams)
assert metadata.default_sampling_params.task == deploy.stages[-1].default_pooling_params["task"]
engine_args = build_engine_args_dict_from_omni_stage_config(
stage,
stage.model_config.model or "/models/Qwen3-ForcedAligner-0.6B",
)
assert engine_args["runner"] == "pooling"
assert engine_args["pooling_output_decoder"] == POOLING_OUTPUT_DECODER_PATH
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.config.endpoint_policy import OmniServingCapability
from vllm_omni.config.stage_config import StageExecutionType, load_deploy_config
from vllm_omni.diffusion.registry import _DIFFUSION_MODELS
from vllm_omni.model_executor.models.mammoth_moda2.pipeline import (
MAMMOTH_MODA2_AR_PIPELINE,
MAMMOTH_MODA2_PIPELINE,
)
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_mammothmoda2_generation_stage_uses_shared_diffusion_runtime() -> None:
ar_stage, dit_stage = MAMMOTH_MODA2_PIPELINE.stages
assert ar_stage.execution_type is StageExecutionType.LLM_AR
assert dit_stage.execution_type is StageExecutionType.DIFFUSION
assert dit_stage.model_arch == "MammothModa2DiTPipeline"
assert dit_stage.custom_process_input_func.endswith(".ar2diffusion")
assert dit_stage.omni_kv_config == {"need_recv_cache": False}
assert dit_stage.input_sources == (0,)
assert dit_stage.final_output is True
assert dit_stage.final_output_type == "image"
def test_mammothmoda2_dit_is_registered_only_with_diffusion_runtime() -> None:
assert _DIFFUSION_MODELS["MammothModa2DiTPipeline"] == (
"mammoth_moda2",
"pipeline_mammothmoda2_dit",
"MammothModa2DiTPipeline",
)
assert "MammothModa2DiTPipeline" not in _OMNI_MODELS
def test_mammothmoda2_deploy_does_not_shadow_request_sampling_fields() -> None:
deploy = load_deploy_config(get_deploy_config_path("mammoth_moda2.yaml"))
assert deploy.stages[1].default_sampling_params is None
def test_mammothmoda2_rejects_unsupported_image_edits() -> None:
restrictions = {
restriction.capability: restriction.reason for restriction in MAMMOTH_MODA2_PIPELINE.endpoint_restrictions
}
assert set(restrictions) == {OmniServingCapability.IMAGE_EDITS}
assert "img2img" in restrictions[OmniServingCapability.IMAGE_EDITS]
assert MAMMOTH_MODA2_AR_PIPELINE.endpoint_restrictions == ()
def test_mammothmoda2_ar_only_topology_is_unchanged() -> None:
assert len(MAMMOTH_MODA2_AR_PIPELINE.stages) == 1
stage = MAMMOTH_MODA2_AR_PIPELINE.stages[0]
assert stage.execution_type is StageExecutionType.LLM_AR
assert stage.final_output_type == "text"
+402 -66
View File
@@ -4,6 +4,7 @@
from __future__ import annotations
import warnings
from dataclasses import fields
from inspect import Parameter, signature
from multiprocessing.reduction import ForkingPickler
@@ -11,6 +12,7 @@ from pathlib import Path
import msgspec
import pytest
import torch
from pydantic import ValidationError
from pydantic.fields import FieldInfo
from transformers import Qwen3OmniMoeConfig
@@ -23,6 +25,7 @@ from vllm.config import SchedulerConfig as VllmSchedulerConfig
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.config import omni_config as omni_config_module
from vllm_omni.config.config_factory import StageConfigFactory
from vllm_omni.config.omni_config import (
BaseVllmOmniStageConfig,
OmniStageCacheConfig,
@@ -47,8 +50,10 @@ from vllm_omni.config.stage_config import (
StageDeployConfig,
StageExecutionType,
StagePipelineConfig,
_apply_platform_overrides,
load_deploy_config,
merge_pipeline_deploy,
resolve_deploy_yaml,
)
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
from vllm_omni.engine.stage_engine_startup import _serialize_stage_config
@@ -59,6 +64,24 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_DEPLOY_DIR = Path(__file__).parents[2] / "vllm_omni" / "deploy"
@pytest.mark.parametrize("async_chunk", [False, True])
def test_native_kv_transfer_requires_completed_ar_stage(async_chunk):
from types import SimpleNamespace
pipeline = SimpleNamespace(stages=(), model_type="test")
deploy = DeployConfig(
async_chunk=async_chunk,
stages=[
StageDeployConfig(stage_id=0, engine_extras={"kv_transfer_config": {"kv_connector": "MooncakeConnector"}})
],
)
if async_chunk:
with pytest.raises(ValueError, match="requires async_chunk=False"):
omni_config_module._validate_async_chunk_support(pipeline, deploy)
else:
omni_config_module._validate_async_chunk_support(pipeline, deploy)
@pytest.fixture(autouse=True)
def _stable_test_platform(monkeypatch):
from vllm_omni import platforms
@@ -94,6 +117,20 @@ def _from_pipeline_key(
)
def test_mammothmoda2_diffusion_stage_projects_native_backend_config() -> None:
config = _from_pipeline_key(
"mammoth_moda2",
cli_overrides={"model": "/models/MammothModa2-Preview"},
)
stage = config.stage_by_id(1)
assert isinstance(stage, VllmOmniDiffusionStageConfig)
assert stage.diffusion_config.model_class_name == "MammothModa2DiTPipeline"
assert stage.diffusion_config.model == "/models/MammothModa2-Preview"
assert stage.diffusion_config.step_execution is False
assert stage.scheduler_config.max_num_seqs == 1
assert stage.connector_config.omni_kv_config == {"need_recv_cache": False}
def test_minimax_h3_text_encoder_tp_targets_only_structured_stage_zero() -> None:
config = _from_pipeline_key(
"minimax_h3_disaggregated",
@@ -123,6 +160,42 @@ def test_non_duplex_deploy_keeps_model_session_capacity_at_one(tmp_path: Path) -
assert [stage.model_config.session_mode for stage in omni_config.stage_configs] == ["turn", "turn"]
def test_nested_stage_override_deep_merges_structured_model_config() -> None:
config = _from_pipeline_key(
"cosmos3_policy",
deploy_config_path=get_deploy_config_path("cosmos3_policy_droid.yaml"),
cli_overrides={"stage_0_model_config": {"guardrails": False}},
)
model_config = config.stage_by_id(0).diffusion_config.model_config
assert model_config["guardrails"] is False
assert model_config["policy_server_config"]["action_space"] == "joint_position"
def _build_single_diffusion_config(
*,
cli_overrides: dict | None = None,
engine_extras: dict | None = None,
) -> VllmOmniConfig:
pipeline = PipelineConfig(
model_type="diffusion-owner-test",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="diffusion",
execution_type=StageExecutionType.DIFFUSION,
final_output=True,
),
),
)
deploy_stages = [StageDeployConfig(stage_id=0, engine_extras=engine_extras)] if engine_extras is not None else []
return VllmOmniConfig.from_pipeline_config(
pipeline,
user_deploy_config=DeployConfig(async_chunk=False, stages=deploy_stages),
cli_overrides=cli_overrides,
)
@pytest.mark.parametrize("model_type", sorted(OMNI_PIPELINES))
def test_vllm_omni_config_from_pipeline_config_matches_merge_pipeline_deploy(model_type: str):
pipeline = _resolve_pipeline_or_skip(model_type)
@@ -219,7 +292,7 @@ def test_from_pipeline_config_normalizes_stage_engine_extras_without_expanding_s
def test_frontend_log_stats_flag_is_not_an_unowned_stage_argument(disabled):
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
config = _from_pipeline_key("dots_tts", cli_overrides={"disable_log_stats": disabled})
config = _from_pipeline_key("voxcpm2", cli_overrides={"disable_log_stats": disabled})
assert config.stage_configs
engine_args = build_engine_args_dict_from_omni_stage_config(config.stage_by_id(0), model="test-model")
assert "disable_log_stats" not in engine_args
@@ -242,6 +315,30 @@ def test_from_pipeline_config_applies_cli_overrides_without_stage_config_runtime
assert stage1.runtime_config.num_gpus == stage1.parallel_config.world_size
def test_diffusion_cli_parallel_overrides_beat_nested_deploy_parallel_config():
"""Flat CLI parallel flags override the deploy YAML's nested parallel_config.
Regression for the NPU nightly failure (#7778): hunyuan_image3_dit.yaml's
platform section sets nested parallel_config.tensor_parallel_size=4 while
the perf tests pass --tensor-parallel-size 2. The flat CLI value must win,
mirroring StageConfig.to_omegaconf, or the stage demands more devices than
the machine has (tp=4 x usp=2 = 8 on a 4-card box).
"""
omni_config = VllmOmniConfig.from_pipeline_config(
_resolve_pipeline_or_skip("hunyuan_image3_dit"),
user_deploy_config=DeployConfig(
stages=[StageDeployConfig(stage_id=0, engine_extras={"parallel_config": {"tensor_parallel_size": 4}})]
),
cli_overrides={"tensor_parallel_size": 2, "ulysses_degree": 2},
)
stage = omni_config.stage_by_id(0)
assert stage.parallel_config.tensor_parallel_size == 2
assert stage.parallel_config.ulysses_degree == 2
assert stage.parallel_config.world_size == 4
@pytest.mark.parametrize(
"cli_overrides",
[
@@ -424,7 +521,7 @@ def test_from_pipeline_config_normalizes_nested_parallel_config_aliases(field_na
assert "data_parallel_master_ip" in stage.parallel_config._omni_explicit_fields
def test_from_pipeline_config_accepts_diffusion_only_cli_fields_for_diffusion_stage():
def test_from_pipeline_config_keeps_global_kv_cache_dtype_outside_diffusion_stage():
stage = _from_pipeline_key(
"dreamzero",
deploy_config_path="dreamzero_tp1_cfg2",
@@ -432,11 +529,57 @@ def test_from_pipeline_config_accepts_diffusion_only_cli_fields_for_diffusion_st
).stage_by_id(0)
assert isinstance(stage, VllmOmniDiffusionStageConfig)
assert stage.diffusion_config.diffusion_kv_cache_dtype == "fp8"
assert stage.diffusion_config.diffusion_kv_cache_dtype is None
def test_stage_cli_field_selection_defers_ownership_validation_until_sources_are_merged():
assert omni_config_module._stage_cli_overrides(0, {"enable_lora": True}) == {"enable_lora": True}
assert omni_config_module._stage_cli_overrides(
0,
{"enable_lora": True},
) == {"enable_lora": True}
@pytest.mark.parametrize(
("cli_overrides", "expected_dtype"),
[
({}, torch.float32),
({"dtype": None}, torch.float32),
({"dtype": "float16"}, torch.float16),
({"dtype": "auto"}, torch.bfloat16),
({"stage_0_dtype": "bfloat16"}, torch.bfloat16),
],
)
def test_diffusion_deploy_dtype_survives_unset_cli_overrides(cli_overrides, expected_dtype):
stage = _build_single_diffusion_config(
engine_extras={"dtype": "float32", "cache_backend": "tea_cache"},
cli_overrides=cli_overrides,
).stage_by_id(0)
assert stage.diffusion_config.dtype == expected_dtype
assert stage.diffusion_config.cache_backend == "tea_cache"
@pytest.mark.parametrize(
"coordination_kwargs",
[
{"omni_master_address": "127.0.0.1", "omni_master_port": 30000},
{"omni_dp_size_local": 2, "omni_heartbeat_timeout": 60.0, "omni_lb_policy": "random"},
],
)
def test_diffusion_ingress_routes_coordination_fields_away(coordination_kwargs):
normalize = omni_config_module.normalize_and_validate_diffusion_engine_ingress_kwargs
assert normalize({**coordination_kwargs, "enable_sleep_mode": True}, stage_id=0) == {"enable_sleep_mode": True}
with pytest.raises(ValueError, match="omni_master_unknown_field"):
normalize({**coordination_kwargs, "omni_master_unknown_field": "unknown"}, stage_id=0)
def test_diffusion_ingress_defers_defaults(monkeypatch):
monkeypatch.setenv("DIFFUSION_CACHE_BACKEND", "tea_cache")
assert omni_config_module.normalize_and_validate_diffusion_engine_ingress_kwargs({}, stage_id=0) == {}
assert omni_config_module.normalize_and_validate_diffusion_engine_ingress_kwargs(
{"dtype": None, "cache_backend": None}, stage_id=0
) == {"dtype": None, "cache_backend": None}
@pytest.mark.parametrize(
@@ -642,6 +785,67 @@ def test_joyai_code2wav_waits_for_full_payload():
assert code2wav.model_config.requires_full_payload_input is True
@pytest.mark.parametrize("model_runner", ["v1", "v2"])
def test_deploy_model_runner_selection_propagates_to_every_stage(tmp_path: Path, model_runner: str):
deploy_path = tmp_path / "qwen3_tts_runner.yaml"
deploy_path.write_text(
f"""\
pipeline: qwen3_tts
model_runner: {model_runner}
async_chunk: true
stages:
- stage_id: 0
- stage_id: 1
"""
)
deploy = load_deploy_config(deploy_path)
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
legacy_stages = merge_pipeline_deploy(pipeline, deploy)
structured = _from_pipeline_key("qwen3_tts", deploy_config_path=str(deploy_path))
expect_v2 = model_runner == "v2"
assert deploy.model_runner == model_runner
assert all(stage.yaml_engine_args["use_v2_model_runner"] is expect_v2 for stage in legacy_stages)
assert all(stage.model_config.use_v2_model_runner is expect_v2 for stage in structured.stage_configs)
@pytest.mark.parametrize("platform", ["npu", "xpu"])
def test_mrv2_fails_fast_on_platforms_without_native_workers(platform: str):
with pytest.raises(NotImplementedError, match="Model Runner V2"):
_apply_platform_overrides(DeployConfig(model_runner="v2"), platform=platform)
@pytest.mark.parametrize(
("default_name", "mrv2_name"),
[
("qwen3_tts.yaml", "qwen3_tts_mrv2.yaml"),
(
"qwen3_tts_high_concurrency.yaml",
"qwen3_tts_high_concurrency_mrv2.yaml",
),
],
)
def test_qwen3_mrv2_profiles_are_explicit_opt_in(default_name: str, mrv2_name: str):
assert load_deploy_config(_DEPLOY_DIR / default_name).model_runner == "v1"
assert load_deploy_config(_DEPLOY_DIR / mrv2_name).model_runner == "v2"
def test_qwen3_tts_mrv2_retunes_do_not_change_default_mrv1_profile():
default = resolve_deploy_yaml(_DEPLOY_DIR / "qwen3_tts_high_concurrency.yaml")
mrv2 = resolve_deploy_yaml(_DEPLOY_DIR / "qwen3_tts_high_concurrency_mrv2.yaml")
default_extra = default["connectors"]["connector_of_shared_memory"]["extra"]
mrv2_extra = mrv2["connectors"]["connector_of_shared_memory"]["extra"]
# B2 batch stays consistent with the graph buckets; V1 profile untouched.
assert default_extra["decode_batch_max_size"] == 1
assert mrv2_extra["decode_batch_max_size"] == 2
assert mrv2_extra["decode_cudagraph_batch_sizes"] == [1, 2]
assert mrv2_extra["code_predictor_prefix_graphs"] is False
assert mrv2["stages"][1]["enforce_eager"] is False
assert default["stages"][1]["enforce_eager"] is True
def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
assert not hasattr(BaseVllmOmniStageConfig, "from_stage_config")
assert not hasattr(BaseVllmOmniStageConfig, "to_legacy_stage_config")
@@ -654,6 +858,7 @@ def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
"load_config",
"cache_config",
"scheduler_config",
"pooling_config",
"connector_config",
"runtime_config",
"parallel_config",
@@ -737,9 +942,11 @@ def test_sub_config_fields_match_structured_scopes():
"limit_mm_per_prompt",
"interleave_mm_strings",
"media_io_kwargs",
"final_output",
"active_stream_window",
"session_mode",
"duplex_max_sessions",
"use_v2_model_runner",
"enable_sleep_mode",
"default_sampling_params",
"subtalker_sampling_params",
@@ -748,6 +955,7 @@ def test_sub_config_fields_match_structured_scopes():
"custom_voice_dir",
"task_type",
"codec_frame_rate_hz",
"supports_native_mrv2_data_plane",
"enforce_eager",
"max_cudagraph_capture_size",
"enable_flashinfer_autotune",
@@ -797,6 +1005,7 @@ def test_sub_config_fields_match_structured_scopes():
assert {f.name for f in fields(OmniStageConnectorConfig)} == {
"async_chunk",
"omni_kv_config",
"kv_transfer_config",
"stage_connector",
"output_connectors",
"input_connectors",
@@ -889,6 +1098,17 @@ def test_structured_llm_stage_registration_payloads_remain_msgpack_transport_saf
msgspec.msgpack.encode(payload)
def test_structured_diffusion_torch_dtype_is_msgpack_transport_safe():
stage_config = _from_pipeline_key(
"hunyuan_image3_dit",
cli_overrides={"dtype": torch.bfloat16},
).stage_by_id(0)
serialized = _serialize_stage_config(stage_config)
assert serialized["diffusion_config"]["dtype"] == "bfloat16"
msgspec.msgpack.encode(serialized)
def test_diffusion_parallel_config_fields_cover_legacy_surface():
from vllm_omni.diffusion.data import DiffusionParallelConfig
@@ -1352,18 +1572,19 @@ def test_diffusion_config_from_kwargs_reuses_legacy_normalization(monkeypatch):
monkeypatch.setenv("DIFFUSION_CACHE_BACKEND", "TEA_CACHE")
monkeypatch.setattr(current_omni_platform, "is_cuda", lambda: True)
cfg = omni_config_module._DiffusionConfigProjection.from_kwargs(
diffusion_attention_backend="flash_attn",
fa_deterministic=True,
kv_cache_dtype="fp8",
kv_cache_skip_steps="0-1",
kv_cache_skip_layers=[2],
static_lora_scale=0.25,
diffusion_kv_mode="paged_scheduler",
diffusion_kv_max_rows_per_request=2,
diffusers_load_kwargs=None,
diffusers_call_kwargs=None,
)
with pytest.warns(FutureWarning):
cfg = omni_config_module._DiffusionConfigProjection.from_kwargs(
diffusion_attention_backend="flash_attn",
fa_deterministic=True,
kv_cache_dtype="fp8",
kv_cache_skip_steps="0-1",
kv_cache_skip_layers=[2],
static_lora_scale=0.25,
diffusion_kv_mode="paged_scheduler",
diffusion_kv_max_rows_per_request=2,
diffusers_load_kwargs=None,
diffusers_call_kwargs=None,
)
assert cfg.diffusion_attention_config.default.backend == "flash_attn"
assert cfg.fa_deterministic is True
@@ -1400,47 +1621,6 @@ def test_diffusion_config_none_values_preserve_dataclass_defaults():
assert config.enable_sleep_mode is False
@pytest.mark.parametrize(
("canonical_key", "alias_key", "canonical_value", "alias_value"),
[
("lora_scale", "static_lora_scale", 0.75, 0.25),
(
"quantization_config",
"diffusion_quantization_config",
{"method": "canonical"},
{"method": "diffusion-alias"},
),
(
"quantization_config",
"quantization",
{"method": "canonical"},
"legacy-alias",
),
("diffusion_kv_cache_dtype", "kv_cache_dtype", "fp8", "fp16"),
("diffusion_kv_cache_skip_steps", "kv_cache_skip_steps", "0-1", "2-3"),
("diffusion_kv_cache_skip_layers", "kv_cache_skip_layers", "1-2", "3-4"),
("streaming_output", "diffusion_streaming_output", False, True),
],
)
def test_diffusion_alias_conflicts_prefer_canonical_key(
canonical_key,
alias_key,
canonical_value,
alias_value,
):
from vllm_omni.diffusion.data import normalize_omni_diffusion_kwargs
normalized = normalize_omni_diffusion_kwargs(
{
canonical_key: canonical_value,
alias_key: alias_value,
}
)
assert normalized[canonical_key] == canonical_value
assert alias_key not in normalized
def test_from_pipeline_config_normalizes_diffusion_config_aliases_from_engine_args(tmp_path, monkeypatch):
from vllm_omni.platforms import current_omni_platform
@@ -1465,10 +1645,11 @@ def test_from_pipeline_config_normalizes_diffusion_config_aliases_from_engine_ar
)
)
stage = _from_pipeline_key(
"dreamzero",
deploy_config_path=str(deploy_path),
).stage_by_id(0)
with pytest.warns(FutureWarning):
stage = _from_pipeline_key(
"dreamzero",
deploy_config_path=str(deploy_path),
).stage_by_id(0)
assert isinstance(stage, VllmOmniDiffusionStageConfig)
assert stage.diffusion_config.diffusion_attention_config.default.backend == "flash_attn"
@@ -1480,8 +1661,11 @@ def test_from_pipeline_config_normalizes_diffusion_config_aliases_from_engine_ar
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
engine_args = build_engine_args_dict_from_omni_stage_config(stage, model="test-model")
od_config = OmniDiffusionConfig.from_kwargs(**engine_args)
engine_args = build_engine_args_dict_from_omni_stage_config(stage, model=str(tmp_path))
diffusion_kwargs = omni_config_module.extract_diffusion_stage_config_kwargs(
engine_args, stage_id=stage.stage_id, include_engine_adapter_metadata=True
)
od_config = OmniDiffusionConfig.from_kwargs(**diffusion_kwargs)
assert od_config.kv_transfer_config.engine_id == "dit-engine-1"
@@ -1634,10 +1818,8 @@ def test_video_output_transport_survives_stage_override_filtering() -> None:
def test_video_output_transport_reaches_default_diffusion_stage() -> None:
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
transport = {"enable_device_postprocess": True}
stages = AsyncOmniEngine._create_default_diffusion_stage_cfg(
stages = StageConfigFactory.create_default_diffusion(
{
"model": "unused",
"model_class_name": "UnknownPipeline",
@@ -1705,3 +1887,157 @@ def test_compact_offload_config_is_validated_during_projection():
"components": ["dit"],
}
)
@pytest.mark.parametrize("pipeline_async", [True, False])
@pytest.mark.parametrize("stage_async", [None, False, True])
def test_stage_async_chunk_opt_out_matches_legacy_config(pipeline_async, stage_async):
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
deploy = DeployConfig(
async_chunk=pipeline_async,
stages=[StageDeployConfig(stage_id=i, async_chunk=stage_async) for i in (0, 1)],
)
config = VllmOmniConfig.from_pipeline_config(pipeline, user_deploy_config=deploy)
legacy = merge_pipeline_deploy(pipeline, deploy)
expected = pipeline_async and stage_async is not False
assert config.stage_by_id(0).connector_config.async_chunk is expected
assert config.stage_by_id(1).connector_config.async_chunk is expected
assert legacy[1].yaml_engine_args["async_chunk"] is expected
assert config.stage_by_id(1).custom_process_input_func == legacy[1].custom_process_input_func
if not expected:
assert config.stage_by_id(1).custom_process_input_func.endswith("talker2code2wav_token_only")
@pytest.mark.parametrize("disabled_stage", [0, 1])
@pytest.mark.parametrize("builder", [merge_pipeline_deploy, VllmOmniConfig.from_pipeline_config])
def test_async_chunk_rejects_mismatched_connector_edge(disabled_stage, builder):
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
deploy = DeployConfig(
async_chunk=True,
stages=[StageDeployConfig(stage_id=disabled_stage, async_chunk=False)],
)
with pytest.raises(ValueError, match="incompatible async_chunk settings on connector edge 0 -> 1"):
if builder is merge_pipeline_deploy:
builder(pipeline, deploy)
else:
builder(pipeline, user_deploy_config=deploy)
@pytest.mark.parametrize("explicit", [False, True])
def test_diffusion_quantization_origin_survives_projection_and_transport(monkeypatch, explicit):
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
from vllm_omni.quantization import build_quant_config
checkpoint = TransformerConfig.from_dict(
{
"quantization_config": {
"quant_method": "mxfp4",
"is_checkpoint_mxfp4_serialized": True,
"w4a8_fallback_steps": [37],
}
}
)
requested = build_quant_config("mxfp4", w4a8_fallback_steps=[]) if explicit else None
cfg = omni_config_module._DiffusionConfigProjection.from_kwargs(
tf_model_config=checkpoint,
quantization_config=requested,
)
assert cfg.quantization_config_is_auto_detected is not explicit
monkeypatch.setattr(OmniDiffusionConfig, "_resolve_master_port", lambda _self: 29500)
monkeypatch.setattr(OmniDiffusionConfig, "enrich_config", lambda self: self.set_tf_model_config(checkpoint))
cfg.enrich_config()
restored = ForkingPickler.loads(ForkingPickler.dumps(cfg))
assert restored.quantization_config_is_auto_detected is not explicit
assert restored.quantization_config.w4a8_fallback_steps == ([] if explicit else [37])
@pytest.mark.parametrize(
"factory",
[
omni_config_module._DiffusionConfigProjection.from_kwargs,
pytest.param(
lambda **kwargs: __import__(
"vllm_omni.diffusion.data",
fromlist=["OmniDiffusionConfig"],
).OmniDiffusionConfig.from_kwargs(**kwargs),
id="runtime-config",
),
],
)
def test_diffusion_config_entrypoints_reject_unknown_none(factory):
with pytest.raises((ValueError, ValidationError), match="enable_sleep_mod"):
factory(enable_sleep_mod=None)
def test_diffusion_config_projection_rejects_unknown_none():
with pytest.raises(ValueError, match="enable_sleep_mod"):
omni_config_module._DiffusionConfigProjection.from_kwargs(enable_sleep_mod=None)
def test_runtime_diffusion_config_rejects_unknown_none():
from vllm_omni.diffusion.data import OmniDiffusionConfig
with pytest.raises(ValueError, match="enable_sleep_mod"):
OmniDiffusionConfig.from_kwargs(enable_sleep_mod=None)
def test_direct_diffusion_aliases_promote_none_and_reject_real_conflicts(monkeypatch):
from vllm_omni.diffusion import data as diffusion_data
from vllm_omni.diffusion.data import OmniDiffusionConfig
monkeypatch.setattr(diffusion_data, "build_quant_config", lambda config: config)
with pytest.warns(FutureWarning) as warnings:
config = OmniDiffusionConfig.from_kwargs(
quantization={"method": "example"},
quantization_config=None,
kv_cache_dtype="fp8",
diffusion_kv_cache_dtype=None,
enable_sleep_mode=False,
request_batch_max_wait_ms=0.0,
)
warning_messages = [str(warning.message) for warning in warnings]
assert any("quantization" in message for message in warning_messages)
assert any("kv_cache_dtype" in message for message in warning_messages)
assert config.quantization_config == {"method": "example"}
assert config.diffusion_kv_cache_dtype == "fp8"
assert config.enable_sleep_mode is False
assert config.request_batch_max_wait_ms == 0.0
with pytest.raises(ValueError, match=r"quantization.*quantization_config"):
OmniDiffusionConfig.from_kwargs(quantization="fp8", quantization_config={"method": "example"})
def test_structured_diffusion_stage_rejects_unknown_engine_extra():
with pytest.raises(ValueError, match=r"stage 0.*enable_sleep_mod"):
_build_single_diffusion_config(engine_extras={"enable_sleep_mod": None})
with pytest.raises(ValueError, match=r"stage 0.*enable_sleep_mod"):
_build_single_diffusion_config(cli_overrides={"enable_sleep_mod": None})
with pytest.raises(ValueError, match=r"stage 0.*enable_sleep_mod"):
_build_single_diffusion_config(cli_overrides={"stage_0_enable_sleep_mod": None})
with pytest.raises(ValueError, match=r"no structured config owner: parallel_config\.unknown_field"):
_build_single_diffusion_config(engine_extras={"parallel_config": {"unknown_field": 1}})
def test_structured_diffusion_stage_keeps_shared_globals_outside_diffusion():
with warnings.catch_warnings(record=True) as caught:
stage = _build_single_diffusion_config(
cli_overrides={"kv_cache_dtype": "fp8", "seed": 7},
).stage_by_id(0)
assert stage.diffusion_config.diffusion_kv_cache_dtype is None
assert not any(issubclass(warning.category, FutureWarning) for warning in caught)
@pytest.mark.parametrize(
("field_name", "config_kwargs"),
[
("seed", {"cli_overrides": {"stage_0_seed": None}}),
("kv_cache_dtype", {"engine_extras": {"kv_cache_dtype": "fp8"}}),
],
)
def test_structured_diffusion_stage_rejects_explicit_shared_engine_field(field_name, config_kwargs):
with pytest.raises(ValueError, match=rf"stage 0.*{field_name}"):
_build_single_diffusion_config(**config_kwargs)
+51
View File
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Check the sleep/wake E2E topology before loading large BAGEL weights."""
import pytest
from tests.entrypoints.test_omni_sleep_mode import _sleep_deploy_config
from vllm_omni.config.omni_config import (
VllmOmniConfig,
VllmOmniDiffusionStageConfig,
extract_diffusion_stage_config_kwargs,
)
from vllm_omni.config.pipeline_registry import resolve_pipeline_config
from vllm_omni.config.stage_config import load_deploy_config
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize("diffusion_only,tp_size", [(False, 1), (False, 2), (True, 2)])
def test_sleep_deploy_resolves_expected_stages(diffusion_only, tp_size):
deploy = load_deploy_config(_sleep_deploy_config(diffusion_only=diffusion_only, tp_size=tp_size))
pipeline = resolve_pipeline_config(deploy.pipeline or "bagel")
config = VllmOmniConfig.from_pipeline_config(pipeline, user_deploy_config=deploy)
assert len(config.stage_configs) == (1 if diffusion_only else 2)
for stage in config.stage_configs:
is_diffusion = diffusion_only or stage.stage_id == 1
assert isinstance(stage, VllmOmniDiffusionStageConfig) == is_diffusion
# Keep BAGEL's batch budget: a single multimodal item needs 8625 tokens.
assert stage.scheduler_config.max_num_batched_tokens == 32768
assert stage.model_config.enable_sleep_mode is True
assert stage.parallel_config.tensor_parallel_size == tp_size
assert stage.runtime_config.devices == ",".join(str(stage.stage_id * tp_size + rank) for rank in range(tp_size))
if not is_diffusion:
assert stage.cache_config.gpu_memory_utilization == 0.8
if is_diffusion:
engine_args = build_engine_args_dict_from_omni_stage_config(stage, "test-bagel")
terminal = OmniDiffusionConfig.from_kwargs(
**extract_diffusion_stage_config_kwargs(
engine_args, stage_id=stage.stage_id, include_engine_adapter_metadata=True
)
)
assert terminal.enable_sleep_mode is True
assert terminal.parallel_config.tensor_parallel_size == tp_size
if not diffusion_only:
assert config.stage_by_id(1).stage_pipeline_config.input_sources == (0,)
assert deploy.stages[1].input_connectors["from_stage_0"] == "shared_memory_connector"
@@ -1,14 +1,22 @@
"""Test that OmniGenerationScheduler restores chunk-waiting requests
even when the OmniNewRequestData rewrapping fails.
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
Regression test: if process_pending_chunks() moves requests into
internal deques but restore_queues() is not called due to an exception,
those requests are permanently orphaned.
"""
"""Chunk-lifecycle coverage for OmniGenerationScheduler: restore on error,
native terminal state, in-flight no-resubmission, slot release on finish,
and single scheduling of a terminal empty-prompt chunk."""
from collections import deque
from types import SimpleNamespace
import pytest
import torch
from vllm import SamplingParams
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
from vllm.v1.request import Request, RequestStatus
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler, _has_async_chunk_payload_to_run
from vllm_omni.engine.serialization import serialize_additional_information
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -20,16 +28,20 @@ class FakeAdapter:
self.waiting_for_chunk_waiting_requests = deque()
self.waiting_for_chunk_running_requests = deque()
self.restore_called = False
self.done_request_ids = set()
def process_pending_chunks(self, waiting, running):
"""Simulate moving requests out of the scheduler queues."""
# Move one request from running into internal deque
def process_pending_chunks(self, waiting, running, scheduler_requests=None):
if running:
req = running.pop()
self.waiting_for_chunk_running_requests.append(req)
def is_done_receiving_chunks(self, request_id):
return request_id in self.done_request_ids
def collect_failed_send_request_ids(self):
return {}
def restore_queues(self, waiting, running, scheduler_requests=None):
"""Put requests back."""
self.restore_called = True
running.extend(self.waiting_for_chunk_running_requests)
self.waiting_for_chunk_running_requests = deque()
@@ -38,6 +50,131 @@ class FakeAdapter:
pass
def _make_generation_scheduler(waiting_request, *, use_v2_model_runner=False):
scheduler = OmniGenerationScheduler.__new__(OmniGenerationScheduler)
scheduler.max_num_scheduled_tokens = 8
scheduler.max_num_running_reqs = 1
scheduler._pause_state = PauseState.UNPAUSED
scheduler.running = []
scheduler.waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.waiting.add_request(waiting_request)
scheduler.skipped_waiting = create_request_queue(SchedulingPolicy.FCFS)
scheduler.requests = {waiting_request.request_id: waiting_request}
scheduler.policy = SchedulingPolicy.FCFS
scheduler.chunk_transfer_adapter = FakeAdapter()
scheduler.input_coordinator = None
scheduler.log_stats = False
scheduler.scheduler_config = SimpleNamespace(enable_chunked_prefill=True)
scheduler.num_lookahead_tokens = 0
scheduler.kv_cache_manager = SimpleNamespace(
new_step_starts=lambda: None,
allocate_slots=lambda *args, **kwargs: SimpleNamespace(get_block_ids=lambda: ([1],)),
get_num_common_prefix_blocks=lambda request_id: [0],
take_new_block_ids=lambda: None,
)
scheduler.kv_cache_config = SimpleNamespace(kv_cache_groups=[object()])
scheduler.use_v2_model_runner = use_v2_model_runner
scheduler._retains_state_across_chunks = False
scheduler.needs_kv_cache_zeroing = False
scheduler.finished_req_ids = set()
scheduler.encoder_cache_manager = SimpleNamespace(get_freed_mm_hashes=lambda: [])
scheduler.connector = None
scheduler.ec_connector = None
scheduler.prev_step_scheduled_req_ids = set()
scheduler._pending_finish_reqs = []
scheduler._consume_pending_connector_output = lambda model_mode: None
scheduler._process_pending_input_timeouts = lambda: None
scheduler._make_cached_request_data = lambda **kwargs: SimpleNamespace(
req_ids=[],
resumed_req_ids=[],
new_token_ids=[],
all_token_ids=[],
new_block_ids=[],
num_computed_tokens=[],
num_output_tokens=[],
)
scheduler._update_after_schedule = lambda output: None
scheduler._wrap_omni_scheduler_output = lambda output: output
return scheduler
def _chunk_request(request_id, **kwargs):
defaults = dict(
request_id=request_id,
prompt_token_ids=[1],
num_computed_tokens=0,
num_in_flight_tokens=0,
status=None,
sampling_params=None,
pooling_params=None,
mm_features=None,
lora_request=None,
prompt_is_token_ids=True,
additional_information=None,
external_req_id=request_id,
prefill_stats=None,
record_event=lambda *args, **kwargs: None,
)
defaults.update(kwargs)
return SimpleNamespace(**defaults)
def test_chunk_lifecycle_no_resubmit_and_state_survives_requeue(monkeypatch) -> None:
# Native plane: a completed payload without a new chunk is never executed
# again; requeued requests keep WAITING_FOR_CHUNK and their token counts.
scheduler = _make_generation_scheduler(_chunk_request("unused"))
monkeypatch.setattr("vllm_omni.core.sched.omni_generation_scheduler.create_request_queue", create_request_queue)
scheduler._native_data_plane = True
scheduler.chunk_transfer_adapter = None
scheduler.input_coordinator = SimpleNamespace(finished_requests=set(), restore_queues=lambda *args, **kwargs: None)
scheduler.running = []
scheduler.waiting = create_request_queue(scheduler.policy)
scheduler.skipped_waiting = create_request_queue(scheduler.policy)
completed = Request("completed", [1, 2], SamplingParams(max_tokens=4), pooling_params=None)
completed.status = RequestStatus.RUNNING
completed.num_computed_tokens = 2
scheduler.running = [completed]
scheduler.requests = {"completed": completed}
scheduler.input_coordinator.finished_requests.add("completed") # terminal payload, no new chunk
assert scheduler._is_done_receiving_chunks("completed")
assert not scheduler._is_done_receiving_chunks("unknown")
output = scheduler.schedule()
assert not output.num_scheduled_tokens
assert scheduler._pending_finish_reqs == [completed]
completed.status = RequestStatus.WAITING_FOR_CHUNK
scheduler._pending_finish_reqs = []
scheduler._requeue_completed_native_chunks()
assert completed.status == RequestStatus.WAITING_FOR_CHUNK
assert completed.num_computed_tokens == 2
assert not scheduler.running
def test_generation_scheduler_schedules_terminal_empty_prompt_chunk_once(monkeypatch: pytest.MonkeyPatch) -> None:
waiting = _chunk_request(
"terminal",
prompt_token_ids=[],
num_prompt_tokens=0,
additional_information=serialize_additional_information(
{"codes": {"audio": torch.tensor([[1, 2, 3]], dtype=torch.long)}}
),
)
scheduler = _make_generation_scheduler(waiting)
scheduler.chunk_transfer_adapter.done_request_ids.add("terminal")
scheduler.requests = {"terminal": waiting}
assert _has_async_chunk_payload_to_run(waiting)
# A terminal codec request with an empty prompt is still scheduled exactly once.
monkeypatch.setattr("vllm_omni.core.sched.omni_generation_scheduler.create_request_queue", create_request_queue)
output = OmniGenerationScheduler.schedule(scheduler)
assert output.num_scheduled_tokens == {"terminal": 1}
assert output.scheduled_new_reqs[0].req_id == "terminal"
assert scheduler._pending_finish_reqs == []
class TestRestoreQueuesOnError:
"""Verify that restore_queues is called even when rewrapping raises."""
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for ``OmniARScheduler._drop_aborted_queued_requests``.
``schedule()`` sweeps ``FINISHED_ABORTED`` requests out of the queues before
@@ -68,6 +71,7 @@ def test_schedule_sweeps_skipped_waiting_before_upstream_selection(
aborted = _StubRequest("req-aborted", RequestStatus.FINISHED_ABORTED)
scheduler = _make_scheduler(policy, skipped_waiting=[aborted])
scheduler.chunk_transfer_adapter = None
scheduler.max_num_running_reqs = 8
scheduler.input_coordinator = None
scheduler.num_waiting_for_streaming_input = 0
scheduler._process_pending_omni_inputs = lambda model_mode: None
@@ -7,6 +7,7 @@ window fills with stale entries after K completions.
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
@@ -15,6 +16,7 @@ import pytest
# Request / RequestStatus are bound in this module. Ruff isort would reorder them.
# isort: off
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.v1.request import RequestStatus
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
# isort: on
@@ -72,3 +74,85 @@ def test_free_request_is_safe_without_a_chunk_transfer_adapter():
sched._free_request(request) # must not raise
sched._free_input_coordinator_request.assert_called_once_with("req-1")
def test_native_mooncake_finish_uses_confirmed_token_boundary() -> None:
sched = _make_scheduler(chunk_transfer_adapter=None)
sched.vllm_config = SimpleNamespace(kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector"))
observed = {}
def connector_finished(request):
observed["status"] = request.status
observed["num_computed_tokens"] = request.num_computed_tokens
return True, None
sched._connector_finished = connector_finished
request = _make_finished_request("req-native")
request.client_index = 0
request.status = RequestStatus.FINISHED_STOPPED
request.kv_transfer_params = {
"do_remote_decode": True,
"transfer_id": "xfer-req-native",
}
request.num_computed_tokens = 13
request.num_output_placeholders = 1
request.num_prompt_tokens = 10
kv_params, _ = sched._free_request(request)
assert observed == {
"status": RequestStatus.FINISHED_LENGTH_CAPPED,
"num_computed_tokens": 12,
}
assert request.status == RequestStatus.FINISHED_STOPPED
assert request.num_computed_tokens == 13
assert kv_params == {
"transfer_id": "xfer-req-native",
"num_transfer_tokens": 12,
}
def test_native_mooncake_recycles_ar_pages_in_physical_order() -> None:
sched = OmniARScheduler.__new__(OmniARScheduler)
sched.vllm_config = SimpleNamespace(kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector"))
blocks = [SimpleNamespace(block_id=7), SimpleNamespace(block_id=2), SimpleNamespace(block_id=4)]
sched.kv_cache_manager = SimpleNamespace(
enable_caching=False,
pop_blocks_for_free=MagicMock(return_value=blocks),
block_pool=SimpleNamespace(free_blocks=MagicMock()),
)
sched.defer_block_free = False
sched.processed_step_seq = 3
request = SimpleNamespace(last_sched_seq=3)
sched._free_request_blocks(request)
sched.kv_cache_manager.block_pool.free_blocks.assert_called_once()
freed = sched.kv_cache_manager.block_pool.free_blocks.call_args.args[0]
assert [block.block_id for block in freed] == [2, 4, 7]
def test_native_mooncake_deferred_ar_pages_drain_in_physical_order() -> None:
from collections import deque
sched = OmniARScheduler.__new__(OmniARScheduler)
sched.vllm_config = SimpleNamespace(kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector"))
blocks = [SimpleNamespace(block_id=7), SimpleNamespace(block_id=2), SimpleNamespace(block_id=4)]
sched.kv_cache_manager = SimpleNamespace(
enable_caching=False,
pop_blocks_for_free=MagicMock(return_value=blocks),
block_pool=SimpleNamespace(free_blocks=MagicMock()),
)
sched.defer_block_free = True
sched.processed_step_seq = 2
sched.sched_step_seq = 3
sched.deferred_frees = deque()
request = SimpleNamespace(last_sched_seq=3)
sched._free_request_blocks(request)
assert [block.block_id for block in sched.deferred_frees[0][1]] == [7, 4, 2]
sched.processed_step_seq = 3
sched._drain_deferred_frees()
freed = list(sched.kv_cache_manager.block_pool.free_blocks.call_args.args[0])
assert [block.block_id for block in freed] == [2, 4, 7]
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for the AR sampled-token logprob contract."""
from __future__ import annotations
@@ -93,6 +96,7 @@ class _Request:
self.num_computed_tokens = 0
self.num_in_flight_tokens = 0
self.num_output_placeholders = 0
self.output_token_ids: list[int] = []
# vLLM 0.27 (a0c092ee72): Request gained num_stale_output_tokens to
# track in-flight outputs discarded at preemption/streaming-stop.
self.num_stale_output_tokens = 0
@@ -145,6 +149,14 @@ def _make_scheduler_stub(requests: list[_Request]) -> SimpleNamespace:
kv_cache_manager=SimpleNamespace(take_events=lambda: None),
kv_event_publisher=SimpleNamespace(publish=lambda _events: None),
recompute_kv_load_failures=False,
_native_data_plane=False,
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(
use_v2_model_runner=False,
async_chunk=False,
final_output=True,
)
),
)
for name in _MIXIN_UPDATE_HELPERS:
setattr(scheduler, name, MethodType(getattr(OmniSchedulerMixin, name), scheduler))
@@ -167,10 +179,15 @@ def _bind_request_lifecycle(
scheduler.requests.pop(request.request_id, None)
scheduler.finished_req_ids.add(request.request_id)
scheduler.finished_req_ids_dict[request.client_index].add(request.request_id)
# vLLM 0.26 contract: (kv_xfer_params, ec_xfer_params)
# Current vLLM contract: (kv_xfer_params, ec_xfer_params)
return None, None
scheduler._update_request_with_output = update_request
def update_with_history(request, token_ids):
accepted, stopped = update_request(request, token_ids)
request.output_token_ids.extend(accepted)
return accepted, stopped
scheduler._update_request_with_output = update_with_history
scheduler._process_kv_transfer_trigger = lambda _request, _tokens: False
scheduler._handle_stopped_request = handle_stopped or (lambda _request: True)
scheduler._free_request = free_request
@@ -219,6 +236,7 @@ def test_mid_step_stop_trims_logprob_rows_with_token_ids() -> None:
(output,) = outputs[0].outputs
assert output.new_token_ids == [7]
assert output.num_generation_tokens == 1
assert output.finish_reason is FinishReason.STOP
token_rows = np.asarray(output.new_logprobs.logprob_token_ids)
value_rows = np.asarray(output.new_logprobs.logprobs)
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
import pytest
from vllm_omni.core.sched.omni_ar_scheduler import _should_emit_engine_output
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize(
("async_chunk", "final_output", "use_v2", "stopped", "has_control", "expected"),
[
(True, False, True, False, False, False), # async+v2 suppresses data
(True, False, False, False, False, True), # v1 continues to emit
(True, False, True, True, False, True), # stopped always emits
(True, False, True, False, True, True), # control-only still emits
],
)
def test_async_chunk_intermediate_stage_emits_only_control_outputs(
async_chunk: bool,
final_output: bool,
use_v2: bool,
stopped: bool,
has_control: bool,
expected: bool,
) -> None:
model_config = SimpleNamespace(
async_chunk=async_chunk,
final_output=final_output,
use_v2_model_runner=use_v2,
)
assert (
_should_emit_engine_output(
model_config,
stopped=stopped,
has_control=has_control,
)
is expected
)
@@ -17,6 +17,7 @@ import pytest
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.sampling_params import SamplingParams
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.engine import FinishReason
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
@@ -1261,3 +1262,47 @@ def test_context_overflow_emits_an_error_output_with_the_reason() -> None:
sched._emit_streaming_context_overflow_outputs(outputs)
assert len(outputs[2]) == 1
@pytest.mark.parametrize("native", [True, False])
def test_async_chunk_reserves_parked_slots_during_ar_admission(monkeypatch, native) -> None:
sched = _make_scheduler(stage_id=1)
parked = SimpleNamespace(request_id="parked")
sched.requests = {"parked": parked}
sched.waiting = []
sched.running = []
sched._native_data_plane = native
sched.use_v2_model_runner = native
sched.max_num_running_reqs = 8
sched.input_coordinator = (
SimpleNamespace(_waiting_for_chunk_running=[parked], restore_queues=lambda _w, _r: None) if native else None
)
sched.chunk_transfer_adapter = (
None
if native
else SimpleNamespace(
waiting_for_chunk_running_requests=[parked],
_held_non_active=[],
process_pending_chunks=lambda *_a, **_kw: None,
collect_failed_send_request_ids=lambda: {},
restore_queues=lambda *_a, **_kw: None,
postprocess_scheduler_output=lambda *_a, **_kw: None,
)
)
sched._consume_pending_connector_output = lambda model_mode: None
sched._process_pending_input_timeouts = lambda: None
sched._should_defer_waiting_admission = lambda: False
sched.get_finished_requests_needing_kv_transfer = lambda: {}
sched._wrap_omni_scheduler_output = lambda output, **_kwargs: output
observed_limits: list[int] = []
def fake_schedule(self, _throttle_prefills=False):
observed_limits.append(self.max_num_running_reqs)
return SimpleNamespace(scheduled_new_reqs=[])
monkeypatch.setattr(VLLMScheduler, "schedule", fake_schedule)
sched.schedule()
assert observed_limits == [7 if native else 8]
assert sched.max_num_running_reqs == 8
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
from types import SimpleNamespace
@@ -141,6 +144,7 @@ def test_voxcpm2_unified_decode_graph_does_not_defer_without_cuda_graph(monkeypa
def test_unified_decode_graph_deferral_restores_waiting_queue(monkeypatch) -> None:
scheduler = _make_scheduler()
scheduler.max_num_running_reqs = 8
scheduler.running = [_MockRequest("decode")]
original_waiting_req = _MockRequest("waiting", status=RequestStatus.WAITING)
deferred_by_upstream = _MockRequest("deferred-by-upstream", status=RequestStatus.WAITING)
@@ -123,6 +123,9 @@ def _make_fast_path_sched(*, defer_block_free: bool) -> tuple[MagicMock, Request
sched = MagicMock()
sched.max_num_scheduled_tokens = 8
sched.max_num_running_reqs = 8
sched._native_data_plane = False
sched._retains_state_across_chunks = False
sched._pause_state = PauseState.UNPAUSED
sched.requests = {request.request_id: request}
sched.running = [request]
@@ -1,5 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from collections import defaultdict
from types import SimpleNamespace
from typing import Any
import pytest
from vllm.v1.engine import FinishReason
@@ -87,7 +91,7 @@ def test_full_payload_coordinator_matches_legacy_gate(monkeypatch, stage_id, asy
def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
calls = []
calls: list[tuple[Any, ...]] = []
scheduler = _Scheduler()
scheduler.waiting = ["waiting"]
scheduler.running = ["running"]
@@ -115,7 +119,7 @@ def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
collect_failed_send_request_ids=_collect_failed_sends,
)
scheduler.input_coordinator = SimpleNamespace(
restore_queues=lambda waiting: calls.append(("restore-full", waiting))
restore_queues=lambda waiting, running: calls.append(("restore-full", waiting, running))
)
scheduler._process_pending_omni_inputs("ar")
@@ -130,7 +134,7 @@ def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
("chunk-timeouts", omni_scheduler_mixin.DEFAULT_INPUT_WAIT_TIMEOUT_S),
("failed-sends",),
("restore-chunks", scheduler.waiting, scheduler.running, scheduler.requests),
("restore-full", scheduler.waiting),
("restore-full", scheduler.waiting, scheduler.running),
]
@@ -0,0 +1,38 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Drain ready events once, merging live requests and filtering cancellations."""
from types import SimpleNamespace
import pytest
from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin
from vllm_omni.outputs import OmniConnectorOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_ready_inbox_coalesces_live_events_and_drops_cancelled(mocker):
scheduler = OmniSchedulerMixin()
scheduler.requests, scheduler.waiting, scheduler.running = {"r": object()}, [], []
coordinator = SimpleNamespace(
_async_chunk=True, update_request_metadata=mocker.Mock(), process_pending_chunks=mocker.Mock()
)
scheduler.input_coordinator = coordinator
scheduler._init_omni_connector_output_inbox()
scheduler.enqueue_omni_connector_output(OmniConnectorOutput(chunk_ready_req_ids={"r"}))
scheduler.enqueue_omni_connector_output(
OmniConnectorOutput(
chunk_ready_req_ids={"r", "aborted"},
chunk_finished_req_ids={"r", "aborted"},
request_metadata={"r": {"decode_token_end": 2}, "aborted": {"decode_token_end": 99}},
)
)
scheduler._consume_pending_connector_output(model_mode="ar")
coordinator.update_request_metadata.assert_called_once_with(
scheduler.requests, {"r": {"decode_token_end": 2}}, model_mode="ar"
)
coordinator.process_pending_chunks.assert_called_once_with([], [], {"r"}, {"r"})
scheduler._consume_pending_connector_output(model_mode="ar")
assert coordinator.update_request_metadata.call_count == 1
coordinator.process_pending_chunks.assert_called_with([], [], set(), set())
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for the ``num_waiting_for_streaming_input`` resync placed in
``OmniARScheduler.finish_requests`` / ``OmniGenerationScheduler.finish_requests``
and in ``OmniARScheduler.schedule``.
@@ -128,6 +131,7 @@ def test_ar_schedule_resyncs_before_delegating_upstream(monkeypatch: pytest.Monk
scheduler.skipped_waiting = []
scheduler.num_waiting_for_streaming_input = 1 # leaked by the status rewrite
scheduler.policy = "fcfs"
scheduler.max_num_running_reqs = 8
seen: dict[str, int] = {}
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for OmniSchedulingCoordinator.
These tests use mock request objects and mock queues. They do not require
@@ -20,7 +20,9 @@ import torch
import vllm_omni.core.sched.omni_scheduling_coordinator as coord_mod
from vllm_omni.core.sched.omni_scheduling_coordinator import (
OmniSchedulingCoordinator,
uses_native_mrv2_data_plane,
)
from vllm_omni.core.sched.output import OmniChunkRecvHandle
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -56,6 +58,7 @@ def _make_request(req_id: str, status: str = "waiting") -> SimpleNamespace:
prompt_token_ids=[],
num_prompt_tokens=0,
num_computed_tokens=0,
num_output_placeholders=0,
_all_token_ids=[],
_output_token_ids=[],
payload_sender_info=None,
@@ -83,7 +86,7 @@ class MockQueue:
def prepend_requests(self, requests):
self._items = list(requests) + self._items
def remove(self, request):
def remove_request(self, request):
self._items.remove(request)
def remove_requests(self, requests):
@@ -96,7 +99,47 @@ class MockQueue:
# ------------------------------------------------------------------ #
class TestCoordinatorUpdateRequestMetadata(unittest.TestCase):
class TestNativeMRV2DataPlaneSelection(unittest.TestCase):
def test_native_plane_requires_v2_and_async_chunk_capability(self):
native = SimpleNamespace(async_chunk=True, supports_native_mrv2_data_plane=True)
self.assertTrue(uses_native_mrv2_data_plane(native, use_v2_model_runner=True))
self.assertFalse(uses_native_mrv2_data_plane(native, use_v2_model_runner=False))
self.assertFalse(uses_native_mrv2_data_plane(SimpleNamespace(async_chunk=False), use_v2_model_runner=True))
def test_chunk_registration_ready_and_terminal_lifecycle():
coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
req = _make_request("internal", status=RequestStatus.WAITING)
req.external_req_id = "external"
waiting = MockQueue([req])
coord.process_pending_chunks(waiting, [], set(), set())
assert req.status == RequestStatus.WAITING_FOR_CHUNK
[handle] = coord.pending_chunk_registrations
assert isinstance(handle, OmniChunkRecvHandle)
assert (handle.request_id, handle.external_req_id) == ("internal", "external")
coord.restore_queues(waiting, [])
coord.process_pending_chunks(waiting, [], {"internal"}, set())
assert req.status == RequestStatus.WAITING
assert "internal" in coord.requests_with_ready_chunks
req.status = RequestStatus.WAITING_FOR_CHUNK
coord.process_pending_chunks(waiting, [], set(), {"internal"})
assert "internal" in coord.finished_requests
def test_chunk_waiting_removes_request_from_running_list():
coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
req = _make_request("running", status=RequestStatus.RUNNING)
running = [req]
coord.process_pending_chunks(MockQueue(), running, set(), set())
assert running == []
assert req.status == RequestStatus.WAITING_FOR_CHUNK
assert list(coord._waiting_for_chunk_running) == [req]
class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase):
"""Test update_request_metadata applies scheduling metadata to requests."""
def test_ar_mode_no_longer_sets_additional_information(self):
@@ -382,17 +425,18 @@ class TestTimeoutDetection(unittest.TestCase):
self.assertEqual(result, {"r1"})
self.assertEqual(len(coord._waiting_for_input), 0)
def test_free_finished_request_clears_waiting_since(self):
"""free_finished_request clears coordinator lifecycle markers."""
coord = OmniSchedulingCoordinator(stage_id=1)
coord._waiting_since["r1"] = 0.0
coord._full_payload_input_received.add("r1")
def test_free_finished_request_clears_all_lifecycle_state(self):
"""free_finished_request makes stale connector events harmless."""
coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
r1, r2 = _make_request("r1"), _make_request("r2")
coord.finished_requests.add("r1")
coord.requests_with_ready_chunks.add("r1")
coord._waiting_for_chunk_running.extend([r1, r2])
coord.pending_chunk_registrations = [OmniChunkRecvHandle(request_id="r1"), OmniChunkRecvHandle(request_id="r2")]
coord.free_finished_request("r1")
self.assertNotIn("r1", coord._waiting_since)
self.assertNotIn("r1", coord._full_payload_input_received)
self.assertNotIn("r1", coord.finished_requests)
if __name__ == "__main__":
unittest.main()
self.assertNotIn("r1", coord.requests_with_ready_chunks)
self.assertEqual([r.request_id for r in coord._waiting_for_chunk_running], ["r2"])
self.assertEqual([h.request_id for h in coord.pending_chunk_registrations], ["r2"])
File diff suppressed because it is too large. Load diff
-119
View File
@@ -1,119 +0,0 @@
import pytest
import torch
from vllm_omni.core.prefix_cache import OmniTensorPrefixCache
# `cpu` is the lane selector, not a hardware claim: the only Buildkite steps that
# reach tests/ select `core_model and cpu`, and they run on GPU hardware (l4_1).
# Marking this `cuda` collects it nowhere. Real hardware need is the skipif below.
pytestmark = [
pytest.mark.core_model,
pytest.mark.cpu,
pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a CUDA device"),
]
NUM_BLOCKS = 10
BLOCK_SIZE = 4
HIDDEN_SIZE = 2
DTYPE = torch.float32
def get_omni_pcache() -> OmniTensorPrefixCache:
"""Build an OmniTensorPrefixCache, but don't init mm tensors."""
return OmniTensorPrefixCache(
num_blocks=NUM_BLOCKS,
block_size=BLOCK_SIZE,
hidden_size=HIDDEN_SIZE,
hs_dtype=DTYPE,
)
def test_drain_ready_async_writes_is_a_noop_before_the_event_is_ready(monkeypatch):
"""drain_ready_async_writes() must not touch the cache while the
scheduled D2H's event has not completed, and must consume it exactly
once after it has.
Real GPU timing (e.g. via torch.cuda._sleep to stall the copy stream)
was tried and found unreliable on this box: cross-stream wait_stream/
wait_event did not reliably delay event.query() from reporting ready
immediately, likely a virtualization/driver quirk of this cloud GPU.
Forcing the event's query() result directly tests the branching logic
in drain_ready_async_writes/_consume_pending_write deterministically,
independent of real CUDA scheduling behavior.
"""
cache = get_omni_pcache()
mm_key = "codes.audio"
cache.maybe_init_missing_mm_cache_keys({mm_key: torch.zeros(4, HIDDEN_SIZE)}, seq_len=4)
num_tokens = 4
slot_mapping_gpu = torch.arange(8, 8 + num_tokens, dtype=torch.int64, device="cuda")
mm_gpu = {
mm_key: torch.arange(num_tokens * HIDDEN_SIZE, dtype=torch.float32, device="cuda").reshape(
num_tokens, HIDDEN_SIZE
)
}
cache.schedule_async_write(
hidden_states_gpu=None,
multimodal_outputs_gpu=mm_gpu,
slot_mapping_gpu=slot_mapping_gpu,
num_tokens_unpadded=num_tokens,
num_tokens_padded=num_tokens,
)
pending_event = cache._pending_write.event
rows = cache.mm_outputs_cache[mm_key].view(-1, HIDDEN_SIZE)
monkeypatch.setattr(pending_event, "query", lambda: False)
assert cache.drain_ready_async_writes() == 0
assert torch.all(rows[8 : 8 + num_tokens] == 0)
assert cache._pending_write is not None
monkeypatch.setattr(pending_event, "query", lambda: True)
assert cache.drain_ready_async_writes() == 1
assert torch.equal(rows[8 : 8 + num_tokens], mm_gpu[mm_key].cpu())
assert cache._pending_write is None
# Nothing pending anymore: a second drain is a no-op, not a re-scatter.
assert cache.drain_ready_async_writes() == 0
def test_schedule_async_write_consumes_previous_pending_write_first():
"""Scheduling step N+1's write must first consume (scatter) step N's
pending write, even if the caller never explicitly called
drain_ready_async_writes() in between -- otherwise step N's data would
be silently dropped when overwritten by step N+1's pending write."""
cache = get_omni_pcache()
mm_key = "codes.audio"
cache.maybe_init_missing_mm_cache_keys({mm_key: torch.zeros(4, HIDDEN_SIZE)}, seq_len=4)
num_tokens = 4
first_slots = torch.arange(8, 8 + num_tokens, dtype=torch.int64, device="cuda")
first_gpu = {mm_key: torch.full((num_tokens, HIDDEN_SIZE), 1.0, device="cuda")}
second_slots = torch.arange(12, 12 + num_tokens, dtype=torch.int64, device="cuda")
second_gpu = {mm_key: torch.full((num_tokens, HIDDEN_SIZE), 2.0, device="cuda")}
cache.schedule_async_write(
hidden_states_gpu=None,
multimodal_outputs_gpu=first_gpu,
slot_mapping_gpu=first_slots,
num_tokens_unpadded=num_tokens,
num_tokens_padded=num_tokens,
)
torch.accelerator.synchronize()
# This call's internal _consume_pending_write() must flush the FIRST
# write before the second one is scheduled; the caller never drained.
cache.schedule_async_write(
hidden_states_gpu=None,
multimodal_outputs_gpu=second_gpu,
slot_mapping_gpu=second_slots,
num_tokens_unpadded=num_tokens,
num_tokens_padded=num_tokens,
)
torch.accelerator.synchronize()
cache.drain_ready_async_writes()
rows = cache.mm_outputs_cache[mm_key].view(-1, HIDDEN_SIZE)
assert torch.all(rows[8:12] == 1.0)
assert torch.all(rows[12:16] == 2.0)
@@ -0,0 +1,182 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU tests for PrefixCacheRunnerMixin: one-shot construction, policy
snapshot identity/isolation, and the shared save/materialize gates.
No vLLM and no GPU required (same shim as test_prefix_cache.py)."""
import sys
from pathlib import Path
from types import ModuleType, SimpleNamespace
from typing import Any
import pytest
import torch
try: # pragma: no cover - shim only matters on vllm-less dev machines
import vllm # noqa: F401
except ModuleNotFoundError:
_root = Path(__file__).resolve().parents[2]
for _pkg in ("vllm_omni", "vllm_omni.core"):
if _pkg not in sys.modules:
_m = ModuleType(_pkg)
_m.__path__ = [str(_root / _pkg.replace(".", "/"))]
sys.modules[_pkg] = _m
import logging
_vllm = ModuleType("vllm")
_vllm_logger = ModuleType("vllm.logger")
setattr(_vllm_logger, "init_logger", logging.getLogger)
setattr(_vllm, "logger", _vllm_logger)
sys.modules["vllm"] = _vllm
sys.modules["vllm.logger"] = _vllm_logger
from vllm_omni.core.prefix_cache.interface import PrefixCacheConfig, StageCacheOutputs
from vllm_omni.core.prefix_cache.runner_mixin import PrefixCacheRunnerMixin
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
BLOCK_SIZE = 4
def _patch_pp(monkeypatch, is_last: bool) -> None:
"""Route the mixin's call-time get_pp_group import to a fake rank."""
mod = ModuleType("vllm.distributed.parallel_state")
setattr(mod, "get_pp_group", lambda: SimpleNamespace(is_last_rank=is_last))
if "vllm.distributed" not in sys.modules:
monkeypatch.setitem(sys.modules, "vllm.distributed", ModuleType("vllm.distributed"))
monkeypatch.setitem(sys.modules, "vllm.distributed.parallel_state", mod)
def _full_attention_groups():
"""A single-group config that passes check_prefix_cache_kv_groups.
Real FullAttentionSpec when vllm is installed; otherwise a stand-in
registered under the import path the check resolves at call time.
"""
try:
from vllm.v1.kv_cache_interface import FullAttentionSpec
except ModuleNotFoundError:
iface = sys.modules.get("vllm.v1.kv_cache_interface")
if iface is None:
iface = ModuleType("vllm.v1.kv_cache_interface")
class _FullAttentionSpecStub:
pass
setattr(iface, "FullAttentionSpec", _FullAttentionSpecStub)
sys.modules["vllm.v1"] = ModuleType("vllm.v1")
sys.modules["vllm.v1.kv_cache_interface"] = iface
FullAttentionSpec = getattr(iface, "FullAttentionSpec")
return [SimpleNamespace(kv_cache_spec=FullAttentionSpec.__new__(FullAttentionSpec))]
class _FakeBlockTables:
"""Group-0 table whose layout attrs pass check_prefix_cache_block_layout."""
def __init__(self):
self.block_tables = [object()]
def __getitem__(self, idx):
return SimpleNamespace()
class _Runner(PrefixCacheRunnerMixin):
def __init__(self, cfg: PrefixCacheConfig | None = None):
# Any: tests swap in _CacheStub for the manager.
self.omni_prefix_cache: Any = None
self._omni_prefix_cache_cfg = cfg
self.input_batch = SimpleNamespace(block_table=_FakeBlockTables(), req_ids=[])
self.kv_cache_config = SimpleNamespace(kv_cache_groups=_full_attention_groups())
self.is_pooling_model = False
def _cfg() -> PrefixCacheConfig:
return PrefixCacheConfig(num_blocks=8, block_size=BLOCK_SIZE)
def _sched():
return SimpleNamespace(scheduled_new_reqs=(), finished_req_ids=set(), num_scheduled_tokens={})
class _CacheStub:
def __init__(self):
self.save_calls = []
self.materialize_calls = []
self.outs = StageCacheOutputs(hidden_states=None, mm_outputs={})
def save_outputs(self, hidden, mm, *, num_tokens_unpadded, num_tokens_padded):
self.save_calls.append((hidden, mm, num_tokens_unpadded, num_tokens_padded))
return 7
def materialize(self, step_id, req_ids):
self.materialize_calls.append((step_id, req_ids))
return self.outs
def test_step_begin_builds_once_and_registers_snapshot_policy(monkeypatch):
_patch_pp(monkeypatch, is_last=True)
r = _Runner(_cfg())
r._snapshot_prefix_cache_model_policy(
SimpleNamespace(requires_full_prefix_cached_hidden_states=True, deferred_prefix_cache_mm_keys=("k",))
)
r._prefix_cache_step_begin(_sched())
mgr = r.omni_prefix_cache
assert mgr is not None and r._omni_prefix_cache_cfg is None
assert mgr._policy is r._omni_cache_policy # the load-time snapshot, same object
r._prefix_cache_step_begin(_sched())
assert r.omni_prefix_cache is mgr # one-shot
def test_default_policy_isolated_per_instance():
a, b = _Runner(), _Runner()
a._snapshot_prefix_cache_model_policy(SimpleNamespace(requires_full_prefix_cached_hidden_states=False))
assert a._omni_cache_policy.needs_full_hidden_states is False
assert a._model_needs_full_prefix_hidden_states() is False
assert b._omni_cache_policy.needs_full_hidden_states is True # class default untouched
assert PrefixCacheRunnerMixin._omni_cache_policy.needs_full_hidden_states is True
def test_non_last_rank_consumes_cfg_without_building(monkeypatch):
_patch_pp(monkeypatch, is_last=False)
r = _Runner(_cfg())
r._prefix_cache_step_begin(_sched())
assert r.omni_prefix_cache is None and r._omni_prefix_cache_cfg is None
def test_disabled_cache_step_begin_is_a_noop():
r = _Runner(cfg=None) # enable_prefix_caching off: nothing staged
r._prefix_cache_step_begin(_sched())
assert r.omni_prefix_cache is None
def test_save_step_gates_and_passthrough(monkeypatch):
r = _Runner()
stub = _CacheStub()
hidden = torch.zeros(2, 2)
def save():
return r._prefix_cache_save_step(hidden, None, num_tokens_unpadded=2, num_tokens_padded=2)
_patch_pp(monkeypatch, is_last=True)
assert save() is None # cache off
r.omni_prefix_cache = stub
r.is_pooling_model = True
assert save() is None # pooling stage never writes
r.is_pooling_model = False
_patch_pp(monkeypatch, is_last=False)
assert save() is None # not the last PP rank
_patch_pp(monkeypatch, is_last=True)
assert save() == 7
assert stub.save_calls == [(hidden, {}, 2, 2)] # empty mm stays {}
def test_materialize_requires_explicit_step_and_snapshot_req_ids():
r = _Runner()
stub = _CacheStub()
r.omni_prefix_cache = stub
assert r._prefix_cache_materialize(None, ["a"]) == (None, None)
assert stub.materialize_calls == [] # step_id None: nothing to consume
hidden = {"a": torch.zeros(1, 2)}
stub.outs = StageCacheOutputs(hidden_states=hidden, mm_outputs={})
assert r._prefix_cache_materialize(7, ["a"]) == (hidden, None) # empty mm -> None
assert stub.materialize_calls == [(7, ["a"])]
@@ -54,7 +54,8 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": ["H100", "B200"],
"npu": "A3"
},
"num_cards": 2
}
+81 -5
View File
@@ -26,8 +26,8 @@
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-realtime-tts",
"endpoint": "/v1/realtime",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
32,
64,
@@ -42,7 +42,73 @@
"disable_shuffle": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
"baseline": {
"H100": {
"request_throughput": [
0.9731,
1.1986,
1.686
],
"mean_ttft_ms": [
152.5665,
270.2559,
301.455
],
"mean_e2el_ms": [
1027.2953,
3306.3735,
4686.5366
],
"mean_audio_ttfp_ms": [
458.1553,
2030.2335,
2068.6233
],
"mean_audio_rtf": [
0.2357,
0.791,
1.0324
]
},
"A3": {
"request_throughput": [
0.5383,
0.6042,
0.7547
],
"mean_ttft_ms": [
333.2633,
533.8656,
547.3388
],
"mean_e2el_ms": [
1857.2154,
6600.5101,
10450.6811
],
"mean_audio_ttfp_ms": [
986.4666,
3411.0754,
3352.7515
],
"mean_audio_rtf": [
0.4423,
1.5734,
2.3024
]
}
}
}
]
},
@@ -72,8 +138,8 @@
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-realtime-tts",
"endpoint": "/v1/realtime",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
32,
64,
@@ -88,6 +154,16 @@
"disable_shuffle": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
}
]
@@ -26,8 +26,8 @@
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-realtime-tts",
"endpoint": "/v1/realtime",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
10,
40
@@ -40,7 +40,41 @@
"disable_shuffle": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
"baseline": {
"H100": {
"request_throughput": [
0.9731,
1.1986
],
"mean_ttft_ms": [
152.5665,
270.2559
],
"mean_e2el_ms": [
1027.2953,
3306.3735
],
"mean_audio_ttfp_ms": [
458.1553,
2030.2335
],
"mean_audio_rtf": [
0.2357,
0.791
]
}
}
}
]
}
+10 -6
View File
@@ -570,6 +570,7 @@ def test_omni_tpot_baseline_accepts_measured_finite_sample():
@pytest.mark.parametrize(
("num_tpot_samples", "mean_tpot_ms", "match"),
[
(None, 10.0, "no measurable TPOT samples"),
(0, float("nan"), "no measurable TPOT samples"),
(1, float("nan"), "mean_tpot_ms is not finite"),
],
@@ -577,14 +578,17 @@ def test_omni_tpot_baseline_accepts_measured_finite_sample():
def test_omni_tpot_baseline_rejects_missing_or_nonfinite_sample(num_tpot_samples, mean_tpot_ms, match):
from tests.dfx.perf.scripts.run_benchmark import assert_result
result = {
"completed": 1,
"Hardware": "H100",
"mean_tpot_ms": mean_tpot_ms,
}
if num_tpot_samples is not None:
result["num_tpot_samples"] = num_tpot_samples
with pytest.raises(AssertionError, match=match):
assert_result(
{
"completed": 1,
"Hardware": "H100",
"num_tpot_samples": num_tpot_samples,
"mean_tpot_ms": mean_tpot_ms,
},
result,
{"baseline": {"H100": {"mean_tpot_ms": 20.0}}},
1,
)
@@ -323,12 +323,18 @@ def test_lingbot_like_sink_survives_sliding_window_eviction():
kv = runner.kv_cache
assert kv is not None
for _ in range(8):
state = commit_one_frame(runner, "s1", "main")
sink_block = kv.block_table(state.adapter("main"))[0]
for _ in range(7):
state = commit_one_frame(runner, "s1", "main")
table = kv.block_table(state.adapter("main"))
assert table[0] != kv.null_block_id
assert table[1] == kv.null_block_id
adapter = state.adapter("main")
table = kv.block_table(adapter)
assert len(table) == 6 # one sink + five recent frames
assert table[0] == sink_block
assert kv.null_block_id not in table
assert adapter.absolute_num_computed_tokens == 8 * BLOCK
assert adapter.num_computed_tokens == 6 * BLOCK
ctx = state.get_kv_caches("main", seq_len=BLOCK, commit_current=False)[0].forward_ctx
visible, _ = ctx.video_block_table(torch.device("cpu"))
+103 -1
View File
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for the AR-Diffusion KV cache helpers (Phase 1, PR-2).
Covers the request adapter, the chunk-window spec/manager (registration + the
@@ -269,9 +270,14 @@ def test_compiled_paged_write_does_not_clone_the_pool():
# drives the reinplace pass, so a drift here would silently test nothing.
real_schema = torch.ops.vllm_omni.ar_diffusion_paged_write_attn.default._schema
real_mutated = [arg.name for arg in real_schema.arguments if arg.alias_info and arg.alias_info.is_write]
assert real_mutated == ["key_pool", "value_pool"], (
# The staging buffers are mutated too when reuse_history_staging is on, but they are separate
# allocations and not what this test is about; the pools are what must not be cloned.
assert real_mutated[:2] == ["key_pool", "value_pool"], (
f"the real paged-write op now mutates {real_mutated}; this test still models two pools"
)
assert set(real_mutated) - {"key_pool", "value_pool"} <= {"stage_key", "stage_value"}, (
f"unexpected mutated argument in the real paged-write op: {real_mutated}"
)
num_blocks, heads, dim = 8, 4, 16
cache_shape = (num_blocks, BLOCK, heads, dim)
@@ -357,6 +363,7 @@ def _make_tiny_capacity_kv(
available_bytes: int,
gpu_memory_fraction: float = 1.0,
model_owned_state_bytes_per_session: int = 0,
reuse_history_staging: bool = False,
) -> ARDiffusionKVCache:
return ARDiffusionKVCache(
ARDiffusionKVConfig(
@@ -365,6 +372,7 @@ def _make_tiny_capacity_kv(
window_chunks=3,
sink_chunks=3,
gpu_memory_fraction=gpu_memory_fraction,
reuse_history_staging=reuse_history_staging,
),
num_layers=1,
num_kv_heads=1,
@@ -382,6 +390,33 @@ def _make_tiny_capacity_kv(
)
def test_history_staging_is_reserved_in_the_kv_budget(monkeypatch):
"""The per-layer staging pair is worker-wide memory allocated after admission; the budget must hold it."""
monkeypatch.setenv("VLLM_OMNI_AR_DIFFUSION_KV_GATHER", "1")
# Same geometry as the capacity-two test: 192 bytes fits two sessions exactly without staging.
plain = _make_tiny_capacity_kv(requested_capacity=2, available_bytes=192)
assert plain.session_capacity == 2 and plain.history_staging_reserved_bytes == 0
staged = _make_tiny_capacity_kv(requested_capacity=2, available_bytes=192, reuse_history_staging=True)
# 2 (K, V) x 1 layer x (sink 3 + window 3 + one action-capacity block) tokens x 1 head x 1 dim x 4 bytes.
assert staged.history_staging_reserved_bytes == 2 * 1 * 7 * 1 * 1 * 4
# It comes out of the same budget, so the second session no longer fits.
assert staged.session_capacity == 1
assert staged.num_blocks_total * 8 + staged.history_staging_reserved_bytes <= 192
# Boundary: one session fits at 128 bytes without staging; with staging the same budget is rejected
# up front instead of failing on the first forward.
_make_tiny_capacity_kv(requested_capacity=1, available_bytes=128)
with pytest.raises(ValueError, match="cannot fit one session"):
_make_tiny_capacity_kv(requested_capacity=1, available_bytes=128, reuse_history_staging=True)
def test_history_staging_is_not_reserved_when_the_gather_path_is_off(monkeypatch):
monkeypatch.delenv("VLLM_OMNI_AR_DIFFUSION_KV_GATHER", raising=False)
staged = _make_tiny_capacity_kv(requested_capacity=2, available_bytes=192, reuse_history_staging=True)
assert staged.history_staging_reserved_bytes == 0 and staged.session_capacity == 2
def test_capacity_two_retains_both_windows_and_allocates_next_block():
# LingBot-like single-branch geometry: page=8 bytes; capacity=2 requires
# 17 managed + 3 scratch blocks, plus two 16-byte cross-attention
@@ -588,3 +623,70 @@ def test_non_contiguous_branch_indices_rejected():
kv_branches=(ARDiffusionKVBranchSpec("main", 1),),
session_capacity=1,
)
def test_staged_window_reuse_matches_a_full_gather():
"""Reusing the staged history must be byte-identical to re-gathering the whole window.
This is the property the optimisation rests on: the leading blocks were staged by an earlier forward of
the same AR block, so skipping them can only be correct if what is already there equals what a fresh
gather would write. The test stages once, mutates only the current chunk's blocks in the pool (what a
later probe of the same block does), restages the tail alone, and compares against a full gather.
"""
from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import _stage_window
torch.manual_seed(0)
num_blocks, block_size, heads, dim = 6, 4, 2, 8
current_blocks = 2
key_cache = torch.randn(num_blocks, block_size, heads, dim)
value_cache = torch.randn(num_blocks, block_size, heads, dim)
block_ids = torch.tensor([5, 4, 3, 2, 1, 0])
stage_key = torch.zeros(num_blocks * block_size, heads, dim)
stage_value = torch.zeros(num_blocks * block_size, heads, dim)
_stage_window(stage_key, stage_value, key_cache, value_cache, block_ids, num_blocks, block_size, first_block=0)
# A later probe rewrites only the current chunk's blocks in the pool.
for slot in block_ids[num_blocks - current_blocks :]:
key_cache[slot] = torch.randn(block_size, heads, dim)
value_cache[slot] = torch.randn(block_size, heads, dim)
_stage_window(
stage_key,
stage_value,
key_cache,
value_cache,
block_ids,
num_blocks,
block_size,
first_block=num_blocks - current_blocks,
)
full_key = torch.zeros_like(stage_key)
full_value = torch.zeros_like(stage_value)
_stage_window(full_key, full_value, key_cache, value_cache, block_ids, num_blocks, block_size, first_block=0)
torch.testing.assert_close(stage_key, full_key, rtol=0, atol=0)
torch.testing.assert_close(stage_value, full_value, rtol=0, atol=0)
def test_staged_window_tail_refresh_leaves_history_untouched():
"""The tail restage must not write the history rows -- that is what makes it cheaper."""
from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import _stage_window
torch.manual_seed(1)
num_blocks, block_size, heads, dim = 5, 4, 1, 4
key_cache = torch.randn(num_blocks, block_size, heads, dim)
value_cache = torch.randn(num_blocks, block_size, heads, dim)
block_ids = torch.arange(num_blocks)
stage_key = torch.full((num_blocks * block_size, heads, dim), -1.0)
stage_value = torch.full((num_blocks * block_size, heads, dim), -1.0)
_stage_window(
stage_key, stage_value, key_cache, value_cache, block_ids, num_blocks, block_size, first_block=num_blocks - 1
)
history_rows = (num_blocks - 1) * block_size
assert (stage_key[:history_rows] == -1.0).all(), "history rows were rewritten by a tail restage"
assert (stage_value[:history_rows] == -1.0).all()
torch.testing.assert_close(stage_key[history_rows:], key_cache[-1].reshape(block_size, heads, dim), rtol=0, atol=0)
@@ -6,6 +6,7 @@ import torch
from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec
from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVCache, ARDiffusionKVConfig
from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
DIMS = dict(
@@ -108,3 +109,104 @@ def test_state_close_frees_both_branch_blocks():
state.close()
# All blocks returned — no leak across an evicted session.
assert kv.manager.block_pool.get_num_free_blocks() == free_total
@pytest.mark.parametrize(
"frames_per_call,block_size,max_model_len,num_calls",
[(1, 16, 113, 1000), (3, 16, 113, 1000), (3, 1560, 1 << 20, 241)],
ids=["single-frame", "three-frame", "lingbot-two-minutes"],
)
@pytest.mark.parametrize("sink_chunks,reset_at_boundary", [(0, False), (2, False), (0, True), (2, True)])
def test_rollout_keeps_metadata_bounded(
frames_per_call, block_size, max_model_len, num_calls, sink_chunks, reset_at_boundary
):
"""Long rollouts retain exact global-position K/V with bounded storage indices."""
kv = ARDiffusionKVCache(
ARDiffusionKVConfig(
enable=True,
chunk_size=block_size,
window_chunks=3,
sink_chunks=sink_chunks,
reset_at_boundary=reset_at_boundary,
),
num_layers=1,
num_kv_heads=1,
head_size=4,
dtype=torch.float32,
block_size=block_size,
max_model_len=max_model_len,
available_bytes=1 << 24,
kv_branches=(ARDiffusionKVBranchSpec("main", 0),),
session_capacity=1,
frames_per_block=frames_per_call,
device=torch.device("cpu"),
)
adapter = kv.begin_request("long-rollout")
state = ARDiffusionKVState(kv, "long-rollout", {"main": adapter}, num_layers=1)
free_total = kv.manager.block_pool.get_num_free_blocks()
pool_shape = kv.key_cache(0).shape
retained_frames = sink_chunks + (0 if reset_at_boundary else 3)
for call_index in range(num_calls):
start = call_index * frames_per_call * block_size
stop = start + frames_per_call * block_size
if frames_per_call == 1:
kv.allocate_chunk(adapter)
slots = kv.chunk_write_slots(adapter)
else:
context = state.get_kv_caches("main", seq_len=stop - start, commit_current=True)[0].forward_ctx
context.ensure_video_slots(torch.device("cpu"))
slots = context.current_video_slot_mapping
assert len(kv.block_table(adapter)) <= retained_frames + frames_per_call
values = torch.arange(start, stop, dtype=torch.float32).view(-1, 1, 1).expand(-1, 1, 4)
kv.key_cache(0).flatten(0, 1)[slots] = values
kv.value_cache(0).flatten(0, 1)[slots] = -values
if frames_per_call == 1:
kv.commit_chunk(adapter)
else:
state.commit_paged_context("main")
assert state._committed["main"] == stop
sink_end = min(stop, sink_chunks * block_size)
tail_start = stop if reset_at_boundary else max(sink_end, stop - 3 * block_size)
expected = torch.cat((torch.arange(sink_end), torch.arange(tail_start, stop))).float()
expected = expected.view(-1, 1, 1).expand(-1, 1, 4)
resident = kv.window_block_ids(adapter)
torch.testing.assert_close(kv.key_cache(0)[resident].flatten(0, 1), expected)
torch.testing.assert_close(kv.value_cache(0)[resident].flatten(0, 1), -expected)
assert adapter.completed_chunks * block_size == stop
assert adapter.absolute_num_computed_tokens == stop
assert len(kv.block_table(adapter)) <= retained_frames
assert adapter.num_computed_tokens <= retained_frames * block_size
assert kv.key_cache(0).shape == pool_shape
assert free_total - kv.manager.block_pool.get_num_free_blocks() == min(stop // block_size, retained_frames)
state.reset()
fresh = state.adapter("main")
assert fresh.compacted_tokens == fresh.absolute_num_computed_tokens == fresh.num_computed_tokens == 0
assert kv.block_table(fresh) == []
assert kv.manager.block_pool.get_num_free_blocks() == free_total
kv.allocate_chunk(fresh)
assert len(kv.chunk_write_slots(fresh)) == block_size
state.close()
assert kv.manager.block_pool.get_num_free_blocks() == free_total
def test_compaction_offsets_are_request_local():
kv = make_cache()
free_total = kv.manager.block_pool.get_num_free_blocks()
first, second = kv.begin_request("first"), kv.begin_request("second")
for index in range(8):
kv.allocate_chunk(first)
kv.commit_chunk(first)
if index in (0, 4):
kv.allocate_chunk(second)
kv.commit_chunk(second)
assert first.absolute_num_computed_tokens == 8 * 16
assert first.compacted_tokens == 6 * 16
assert second.absolute_num_computed_tokens == 2 * 16
assert second.compacted_tokens == 0
assert first.num_computed_tokens == second.num_computed_tokens == 2 * 16
assert set(kv.block_table(first)).isdisjoint(kv.block_table(second))
kv.end_request(first)
kv.end_request(second)
group = kv.manager.coordinator.single_type_managers[0]
assert not group.req_to_blocks and not group.num_cached_block
assert kv.manager.block_pool.get_num_free_blocks() == free_total
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for AR-Diffusion paged self-attention contexts."""
from __future__ import annotations
@@ -20,6 +21,8 @@ from vllm_omni.experimental.ar_diffusion.kv_cache import (
ar_diffusion_paged_attention,
paged_write_attn,
)
from vllm_omni.experimental.ar_diffusion.kv_cache import paged_attention as paged_attention_module
from vllm_omni.experimental.ar_diffusion.kv_cache.config import KV_GATHER_ENV
from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
@@ -32,8 +35,12 @@ POS = "positive"
NEG = "negative"
def make_state(*, num_layers=1, window_chunks=2, dtype=torch.float32, device=torch.device("cpu")):
cfg = ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks)
def make_state(
*, num_layers=1, window_chunks=2, dtype=torch.float32, device=torch.device("cpu"), reuse_history_staging=False
):
cfg = ARDiffusionKVConfig(
enable=True, chunk_size=BLOCK, window_chunks=window_chunks, reuse_history_staging=reuse_history_staging
)
kv = ARDiffusionKVCache(
cfg,
num_layers=num_layers,
@@ -236,6 +243,86 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len,
assert st.adapter(POS).completed_chunks == before + (1 if commit_current else 0)
@pytest.mark.parametrize("history_chunks", [0, 1, 2, 3])
def test_staged_reuse_refreshes_the_current_blocks_at_their_live_offset(monkeypatch, history_chunks):
"""A second probe of the same AR block restages its current K/V where the table actually holds it.
The padded table always ends in at least one action-capacity block, and while the window is still
growing in unused window capacity too, so "the last current_blocks entries" is padding. The refresh
has to start right after the visible history: empty (window growing), partial, full, and after a slide.
"""
monkeypatch.setenv(KV_GATHER_ENV, "1")
device = torch.device("cpu")
dtype = torch.float32
kv, st = make_state(dtype=dtype, device=device, window_chunks=2, reuse_history_staging=True)
if history_chunks:
_commit_video_span(kv, st, kv_branch=POS, n_chunks=history_chunks, dtype=dtype, device=device)
ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0].forward_ctx
ctx.ensure_video_slots(device)
key_cache, value_cache = kv.key_cache(0), kv.value_cache(0)
def probe(step: int):
# Each probe of the block writes new current K/V, then stages.
kv._k_pools[0][ctx.current_video_slot_mapping] = torch.full((BLOCK, N_HEADS, HEAD_DIM), float(step))
kv._v_pools[0][ctx.current_video_slot_mapping] = torch.full((BLOCK, N_HEADS, HEAD_DIM), -float(step))
# Store the metadata on the context the way prepare() does.
(ctx.block_table, ctx.query_start_loc, ctx.seq_lens, ctx.max_query_len, ctx.max_seq_len) = (
ctx.build_block_table(action_len=0, query_len=BLOCK, device=device)
)
block_table, max_seq_len = ctx.block_table, ctx.max_seq_len
ctx._prepare_history_staging(0)
n_blocks = max_seq_len // BLOCK
block_ids = block_table[0, :n_blocks].to(torch.long)
stage_k, stage_v = ctx.history_staging(0)
paged_attention_module._stage_window(
stage_k,
stage_v,
key_cache,
value_cache,
block_ids,
n_blocks,
BLOCK,
first_block=ctx.stage_first_block if ctx.reuse_history else 0,
)
full_k = key_cache.index_select(0, block_ids).reshape(n_blocks * BLOCK, N_HEADS, HEAD_DIM)
full_v = value_cache.index_select(0, block_ids).reshape(n_blocks * BLOCK, N_HEADS, HEAD_DIM)
return stage_k, stage_v, full_k, full_v, n_blocks
stage_k, stage_v, full_k, full_v, n_blocks = probe(1)
assert ctx.reuse_history is False
assert torch.equal(stage_k, full_k) and torch.equal(stage_v, full_v)
stage_k, stage_v, full_k, full_v, n_blocks = probe(2)
assert ctx.reuse_history is True
visible_history_blocks = min(history_chunks, 2 - 1) # window of 2 blocks minus the current one
assert ctx.stage_first_block == visible_history_blocks
# The padded table is wider than the live window, so the end-of-table guess is not the offset.
assert n_blocks - 1 != ctx.stage_first_block
assert torch.equal(stage_k, full_k) and torch.equal(stage_v, full_v)
@pytest.mark.parametrize("gather_enabled", [False, True])
def test_staging_is_only_allocated_for_the_gather_path(monkeypatch, gather_enabled):
if gather_enabled:
monkeypatch.setenv(KV_GATHER_ENV, "1")
else:
monkeypatch.delenv(KV_GATHER_ENV, raising=False)
device = torch.device("cpu")
kv, st = make_state(device=device, reuse_history_staging=True)
# The manager owns the pairs and allocates them only when their consumer is on.
assert bool(kv.history_staging) is gather_enabled
ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0].forward_ctx
ctx.ensure_video_slots(device)
(ctx.block_table, ctx.query_start_loc, ctx.seq_lens, ctx.max_query_len, ctx.max_seq_len) = ctx.build_block_table(
action_len=0, query_len=BLOCK, device=device
)
ctx._prepare_history_staging(0)
assert ctx.staging_enabled is gather_enabled
stage_k, _ = ctx.history_staging(0)
assert (stage_k is not None) is gather_enabled
@pytest.mark.skipif(not _gpu_flash_attn_usable(), reason="usable GPU FlashAttention is required")
@pytest.mark.parametrize("history_chunks", [1, 3])
@pytest.mark.parametrize("action_len", [0, 3])
@@ -376,13 +463,27 @@ def test_custom_op_registration_idempotent():
assert hasattr(torch.ops.vllm_omni, "ar_diffusion_paged_write_attn")
def test_custom_op_compiles_fullgraph_without_recompile_on_value_change():
def test_custom_op_mutable_arguments_cannot_be_elided_as_defaults():
# Older PyTorch ADInplaceOrView handlers index positional mutable inputs
# directly. Default-valued trailing inputs can disappear before that handler.
schema = torch.ops.vllm_omni.ar_diffusion_paged_write_attn.default._schema
mutable = [arg for arg in schema.arguments if arg.alias_info is not None and arg.alias_info.is_write]
assert {arg.name for arg in mutable} == {"key_pool", "value_pool", "stage_key", "stage_value"}
assert all(not arg.has_default_value() for arg in mutable)
@pytest.mark.parametrize("reuse_history_staging", [False, True])
def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(monkeypatch, reuse_history_staging):
"""The op must trace as one opaque node: fullgraph OK, and changed tensor
VALUES (new slots / block ids) must not trigger recompilation."""
import torch._dynamo
device = torch.device("cpu")
kv, st = make_state(num_layers=2, window_chunks=2)
monkeypatch.setenv(KV_GATHER_ENV, "1")
kv, st = make_state(num_layers=2, window_chunks=2, reuse_history_staging=reuse_history_staging)
if reuse_history_staging:
# Hold the host-side staging offset fixed while table/slot values change.
_commit_video_span(kv, st, kv_branch=POS, n_chunks=2, dtype=torch.float32, device=device)
def run_one_forward(commit):
contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=commit)
@@ -391,9 +492,8 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change():
q = torch.randn(BLOCK, N_HEADS, HEAD_DIM)
k = torch.randn(BLOCK, N_HEADS, HEAD_DIM)
v = torch.randn(BLOCK, N_HEADS, HEAD_DIM)
# Both layers through ONE compiled fn: layer_idx is a tensor, so a
# different layer must NOT recompile (all 40 DiT blocks share the
# block-forward code object in production).
# Both layers use one compiled function; the tensor-valued layer index
# must not specialize it.
for layer_ctx in contexts:
out = compiled(layer_ctx.to_layer_inputs(), q, k, v)
st.commit_paged_context(POS)
@@ -419,3 +519,52 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change():
# Leave a clean dynamo state for later suites in the same pytest process
# (e.g. model_executor transformers models).
torch._dynamo.reset()
@pytest.mark.skipif(not _gpu_flash_attn_usable(), reason="usable GPU FlashAttention is required")
@pytest.mark.parametrize("history_chunks", [0, 1, 3])
@pytest.mark.parametrize("action_len", [0, 3])
def test_contiguous_kv_gather_path_matches_paged_path_gpu(monkeypatch, history_chunks, action_len):
"""VLLM_OMNI_AR_DIFFUSION_KV_GATHER=1 gathers the visible blocks and runs varlen FA3 without a block table.
Covers an empty, partial and full window plus a partially filled action block:
the gather reads the tail-padding null block, so its (zeroed) rows must be masked.
"""
torch.manual_seed(0)
device = torch.device("cuda")
dtype = torch.float16
kv, st = make_state(dtype=dtype, device=device, window_chunks=2)
if history_chunks:
_commit_video_span(kv, st, kv_branch=POS, n_chunks=history_chunks, dtype=dtype, device=device)
layer_ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0]
ctx = layer_ctx.forward_ctx
current_k = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
current_v = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
action_k = action_v = None
if action_len:
action_k = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
action_v = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
query = torch.randn(1, BLOCK + action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
ctx.prepare(device=device, action_len=action_len, query_len=query.shape[1])
inputs = layer_ctx.to_layer_inputs()
def run() -> torch.Tensor:
return paged_write_attn(
inputs,
query[0],
current_k[0],
current_v[0],
action_k[0] if action_len else None,
action_v[0] if action_len else None,
HEAD_DIM**-0.5,
).unsqueeze(0)
monkeypatch.delenv(KV_GATHER_ENV, raising=False)
paged = run()
monkeypatch.setenv(KV_GATHER_ENV, "1")
gathered = run()
assert torch.isfinite(gathered).all()
# Same kernel family on the same K/V: only accumulation order differs.
torch.testing.assert_close(gathered, paged, rtol=2e-3, atol=2e-3)
+28 -5
View File
@@ -16,13 +16,14 @@ from unittest.mock import Mock
import pytest
import torch
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl
from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl
from vllm_omni.diffusion.attention.backends.utils import fa # noqa: E402
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
is_gpu = current_omni_platform.is_cuda_alike() or current_omni_platform.is_xpu()
HAS_FLASH_ATTN = fa.HAS_FLASH_ATTN
@@ -42,10 +43,12 @@ flash_attn_func = fa.flash_attn_func # noqa: N813
(9, "4", frozenset((2, 3, 4)), 4),
],
)
@pytest.mark.cpu
def test_versioned_flash_attention_selection(device_major, requested, supported, expected):
assert fa._choose_vllm_flash_attn_version(device_major, requested, supported) == expected
@pytest.mark.cpu
def test_versioned_flash_attention_selection_rejects_invalid_or_unavailable_version():
with pytest.raises(ValueError, match="must be 2, 3, or 4"):
fa._choose_vllm_flash_attn_version(9, "5", frozenset((2, 3, 4)))
@@ -92,7 +95,8 @@ def pad_tensor(tensor: torch.Tensor, target_seq_len: int, pad_value: float = 0.0
return torch.cat([tensor, padding], dim=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
def test_padding_equivalence():
"""
Case 1: Test that padded and unpadded inputs produce similar outputs.
@@ -185,7 +189,8 @@ def test_padding_equivalence():
print("✓ Case 1 PASSED: Padded and unpadded outputs are very close!")
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
def test_fa_vs_sdpa():
"""
Case 2: Compare FlashAttention and SDPA backends with padding.
@@ -302,7 +307,8 @@ def test_fa_vs_sdpa():
print("✓ Case 2 PASSED: FA and SDPA outputs are very close!")
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
def test_flash_attn_func_preferred_over_varlen():
"""Test flash_attn_func availability and basic forward call."""
if not HAS_FLASH_ATTN:
@@ -332,7 +338,8 @@ def test_flash_attn_func_preferred_over_varlen():
print("✓ flash_attn_func forward works correctly!")
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
@pytest.mark.parametrize("k_len", [40, 256])
def test_cross_attn_key_padding_vs_sdpa(k_len):
"""
@@ -390,6 +397,7 @@ def test_cross_attn_key_padding_vs_sdpa(k_len):
print("✓ Case 3 PASSED: FA and SDPA cross-attention outputs are very close!")
@pytest.mark.cpu
def test_varlen_masked_routing_by_role(monkeypatch):
"""The unpad route is picked by role, not Q/K length equality; runs without a GPU."""
calls = []
@@ -424,6 +432,7 @@ def test_varlen_masked_routing_by_role(monkeypatch):
assert cu_seqlens_k.tolist() == [0, 2, 5]
@pytest.mark.cpu
def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch):
calls = []
@@ -454,6 +463,7 @@ def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch):
assert calls[0]["softmax_scale"] == 0.5
@pytest.mark.cpu
def test_packed_varlen_metadata_bypasses_mask_unpadding(monkeypatch):
calls = []
@@ -486,6 +496,7 @@ def test_packed_varlen_metadata_bypasses_mask_unpadding(monkeypatch):
assert calls[0][3]["max_seqlen_q"] == 6
@pytest.mark.cpu
def test_packed_varlen_metadata_must_be_complete(monkeypatch):
monkeypatch.setattr(fa, "HAS_FLASH_ATTN", True)
impl = FlashAttentionImpl(num_heads=2, head_size=4, softmax_scale=0.5, causal=False)
@@ -542,6 +553,7 @@ def _fake_torch_npu(monkeypatch, *, npu_fusion_attention=None):
# --- Test group A: boundary resolution (_resolve_packed_seq_npu) -------------
@pytest.mark.cpu
def test_resolve_packed_seq_accepts_real_plus_pad_contract():
impl = _npu_impl()
q = torch.randn(1, 8, 2, 4)
@@ -551,6 +563,7 @@ def test_resolve_packed_seq_accepts_real_plus_pad_contract():
assert impl._resolve_packed_seq_npu(q, q, extra) == ([5, 8], [5, 8])
@pytest.mark.cpu
def test_resolve_packed_seq_single_document_no_padding():
impl = _npu_impl()
q = torch.randn(1, 8, 2, 4)
@@ -574,6 +587,7 @@ def test_resolve_packed_seq_single_document_no_padding():
"pad_longer_than_real",
],
)
@pytest.mark.cpu
def test_resolve_packed_seq_rejects_malformed_metadata(case):
"""Any metadata outside the exact [real, pad] contract returns None so the
caller falls back to the masked path."""
@@ -630,6 +644,7 @@ def _npu_impl_with_mocked_paths() -> tuple[FlashAttentionImpl, torch.Tensor]:
return impl, sentinel
@pytest.mark.cpu
def test_npu_env_unset_uses_varlen(monkeypatch):
monkeypatch.delenv("MINDIE_SD_FA_TYPE", raising=False)
_fake_mindiesd(monkeypatch)
@@ -642,6 +657,7 @@ def test_npu_env_unset_uses_varlen(monkeypatch):
impl._forward_prefix_kv_slice_npu.assert_not_called()
@pytest.mark.cpu
def test_npu_env_laser_uses_slice(monkeypatch):
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
_fake_mindiesd(monkeypatch)
@@ -655,6 +671,7 @@ def test_npu_env_laser_uses_slice(monkeypatch):
@pytest.mark.parametrize("fa_type", ["prompt_flash_attn", "fused_attn_score", "ascend_laser_attenton", "garbage"])
@pytest.mark.cpu
def test_npu_env_non_laser_value_falls_back_to_varlen(monkeypatch, fa_type):
"""Locks in the CURRENT dispatch behavior (PR #5891).
@@ -675,6 +692,7 @@ def test_npu_env_non_laser_value_falls_back_to_varlen(monkeypatch, fa_type):
impl._forward_prefix_kv_slice_npu.assert_not_called()
@pytest.mark.cpu
def test_npu_varlen_opt_in_unset_takes_mask_path(monkeypatch):
"""Models that do not set ``npu_attn_varlen`` (e.g. Wan/Cosmos) keep using
the existing masked attention_forward path; the new branches are skipped."""
@@ -702,6 +720,7 @@ def test_npu_varlen_opt_in_unset_takes_mask_path(monkeypatch):
("query_length", "key_length", "expected_inner_precise"),
[(4, 4, 0), (2, 4, 0), (4, 2, 2)],
)
@pytest.mark.cpu
def test_npu_causal_uses_native_right_down_mode(
monkeypatch,
query_length,
@@ -746,6 +765,7 @@ def test_npu_causal_uses_native_right_down_mode(
assert out is expected_out
@pytest.mark.cpu
def test_npu_causal_composes_explicit_keep_mask(monkeypatch):
expected_out = torch.randn(1, 4, 2, 4)
fusion_attention = Mock(return_value=(expected_out, None, None, None, 0, 0, 0))
@@ -782,6 +802,7 @@ def test_npu_causal_composes_explicit_keep_mask(monkeypatch):
assert out is expected_out
@pytest.mark.cpu
def test_npu_noncausal_without_explicit_mask_stays_unmasked(monkeypatch):
captured: dict = {}
@@ -800,6 +821,7 @@ def test_npu_noncausal_without_explicit_mask_stays_unmasked(monkeypatch):
# --- Test group D: laser input pre-scaling in _forward_prefix_kv_slice_npu --
@pytest.mark.cpu
def test_prefix_kv_slice_applies_laser_input_scaling(monkeypatch):
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
captured: dict = {}
@@ -838,6 +860,7 @@ def test_prefix_kv_slice_applies_laser_input_scaling(monkeypatch):
torch.testing.assert_close(out, torch.ones(1, 8, 2, 4) * 256.0)
@pytest.mark.cpu
def test_prefix_kv_slice_no_scaling_without_factor(monkeypatch):
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
captured: dict = {}
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Cross-repository CPU dispatch contracts, requiring the matching MindIE-SD source.
Only native operators are replaced. Public adapters, rotations, layouts, padding,
scale preparation, sequence metadata and output cropping execute their real code.
Operator substitutes carry floating tensors; these are not numerical quantization tests.
"""
import importlib
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
mindiesd = pytest.importorskip("mindiesd")
torch_npu = importlib.import_module("torch_npu")
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
@pytest.mark.parametrize(
"precision,batch,q_len,kv_len,q_heads",
[
("fp8", 1, 130, 130, 2),
("mxfp8", 1, 130, 130, 2),
("mxfp4", 1, 130, 130, 2),
("mxfp8", 2, 17, 8, 4),
("mxfp8", 2, 8, 17, 4),
],
)
@pytest.mark.parametrize("layout", ["BSND", "BNSD"])
def test_omni_calls_real_dense_public_api(monkeypatch, precision, layout, batch, q_len, kv_len, q_heads):
assert callable(getattr(mindiesd, "quant_attention", None)), "MindIE-SD must export quant_attention"
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
quantized_inputs = []
def dynamic_quant(tensor, **kwargs):
quantized_inputs.append((tensor.clone(), kwargs))
shape = list(tensor.shape)
axis = kwargs.get("axis", -1)
shape[axis] = max(1, shape[axis] // 32)
return tensor, torch.ones(shape, dtype=torch.uint8)
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", dynamic_quant, raising=False)
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", dynamic_quant, raising=False)
execute = Mock(side_effect=lambda query, *args, **kwargs: (query, None))
metadata = Mock(return_value=torch.zeros(1, dtype=torch.int32))
if precision == "fp8":
monkeypatch.setattr(torch.ops.mindiesd, "fused_infer_attention_score_v2", execute, raising=False)
elif precision == "mxfp8":
monkeypatch.setattr(torch_npu, "npu_fused_infer_attention_score_v2", execute, raising=False)
else:
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn", execute, raising=False)
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn_metadata", metadata, raising=False)
query = torch.randn(batch, q_len, q_heads, 64, dtype=torch.bfloat16)
key = torch.randn(batch, kv_len, 2, 64, dtype=torch.bfloat16)
value = torch.randn_like(key)
if layout == "BNSD":
query, key, value = (tensor.transpose(1, 2) for tensor in (query, key, value))
impl = flash_attn.FlashAttentionImpl(
num_heads=q_heads,
head_size=64,
softmax_scale=0.37,
qkv_layout=layout,
)
output = impl.forward_npu(query, key, value, AttentionMetadata(extra={"kv_cache_dtype": precision}))
rotation = get_quant_attention_rotation(query.device, query.dtype, 64)
expected = query @ rotation
torch.testing.assert_close(output, expected)
assert output.shape == query.shape and len(quantized_inputs) == 3
execute.assert_called_once()
kwargs = execute.call_args.kwargs
assert kwargs["softmax_scale"] == 0.37
if precision == "fp8":
assert kwargs["input_layout"] == "BNSD"
assert quantized_inputs[0][0].shape == (2, 130, 64)
assert quantized_inputs[0][1]["row_block_size"] == 128
elif precision == "mxfp8":
args = execute.call_args.args
assert args[0].shape == (batch * q_len, q_heads, 64)
assert args[1].shape == args[2].shape == (batch * kv_len, 2, 64)
assert kwargs["input_layout"] == "TND"
assert kwargs["num_query_heads"] == q_heads and kwargs["num_key_value_heads"] == 2
assert kwargs["actual_seq_qlen"] == [q_len * (i + 1) for i in range(batch)]
assert kwargs["actual_seq_kvlen"] == [kv_len * (i + 1) for i in range(batch)]
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, 0]
else:
seq_axis = 1 if layout == "BSND" else 2
assert quantized_inputs[0][0].shape[seq_axis] == 512
assert kwargs["seqused_q"].tolist() == kwargs["seqused_kv"].tolist() == [512]
assert kwargs["layout_q"] == kwargs["layout_out"] == layout
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
assert kwargs["q_descale_dtype"] == torch_npu.float8_e8m0fnu
assert execute.call_args.args[3].shape[:2] == (1, 2)
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, seq_axis]
assert kwargs["metadata"] is metadata.return_value
metadata.assert_called_once()
@pytest.mark.parametrize("precision", ["fp8", "mxfp4"])
@pytest.mark.parametrize("length", [65, 600])
def test_omni_calls_real_sparse_public_api(monkeypatch, precision, length):
rf = importlib.import_module("mindiesd.layers.flash_attn.sparse_flash_attn_rf_v3")
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
for module in (flash_attn, rainfusion_attn):
monkeypatch.setattr(module, "get_current_diffusion_config_or_none", lambda: None)
rainfusion_attn._mindiesd_supports_precision.cache_clear()
monkeypatch.setattr(rf, "do_tensor_rearrange_only", lambda q, k, v, *a, **kw: (q, k, v))
monkeypatch.setattr(rf, "avgpool", lambda *a, **kw: torch.ones(1))
monkeypatch.setattr(rf, "_generate_mask_direct", lambda *a, **kw: torch.ones(1, 2, 8, 8))
monkeypatch.setattr(rf, "_bsa_inv_rearrange", lambda out, *a: out)
def quantize(tensor, **kwargs):
shape = list(tensor.shape)
axis = kwargs.get("axis", -1)
shape[axis] = max(1, shape[axis] // 32)
return tensor, torch.ones(shape, dtype=torch.uint8)
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", quantize, raising=False)
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", quantize, raising=False)
execute = Mock(side_effect=lambda **kw: (torch.ones_like(kw["query"]), None))
monkeypatch.setattr(torch.ops.mindiesd, "block_sparse_attention", execute, raising=False)
query = torch.randn(1, length, 2, 64, dtype=torch.bfloat16)
impl = rainfusion_attn.RainFusionAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout="BSND",
backend_kwargs={"sparsity": 0.8},
)
plan = rainfusion_attn.RainFusionPlan(used_len=length, prefix_len=0, latent_shape=(1, 1, length))
try:
output = impl._forward_sparse_npu(
query,
query,
query,
plan,
AttentionMetadata(extra={"kv_cache_dtype": precision}),
)
finally:
rainfusion_attn._mindiesd_supports_precision.cache_clear()
torch.testing.assert_close(output, torch.ones_like(query))
execute.assert_called_once()
kwargs = execute.call_args.kwargs
assert kwargs["actual_seq_lengths"] == kwargs["actual_seq_lengths_kv"] == [length]
assert kwargs["q_input_layout"] == "BNSD"
assert kwargs["block_sparse_mask"] is not None
if precision == "mxfp4":
assert kwargs["query"].shape[2] == (length + 63) // 64 * 64
assert kwargs["quant_mode"] == 2
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
assert kwargs["v_scale_dtype"] == torch_npu.float8_e8m0fnu
@@ -0,0 +1,323 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU contract tests for MindIE adapters; no NPU kernels are mocked in production."""
import sys
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm_omni.diffusion.attention import layer as layer_mod
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
_DEFAULT_SKIP_STEPS = frozenset({0, 1, 38, 39})
@pytest.fixture
def runtime(monkeypatch):
mod = ModuleType("mindiesd")
def execute(q, k, v, *, precision, layout, scale, q_rot=None, k_rot=None):
return q
setattr(mod, "quant_attention", Mock(side_effect=execute))
monkeypatch.setitem(sys.modules, mod.__name__, mod)
for module in (flash_attn, layer_mod):
monkeypatch.setattr(module, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(rainfusion_attn, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: False)
supports_precision = rainfusion_attn._mindiesd_supports_precision
supports_precision.cache_clear()
yield mod
supports_precision.cache_clear()
def flash(layout="BSND"):
return flash_attn.FlashAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout=layout,
)
def sparse(**kwargs):
return rainfusion_attn.RainFusionAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout="BSND",
backend_kwargs={"sparsity": 0.8, **kwargs},
)
def video_metadata(**extra):
return AttentionMetadata(
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32), used_len=4096),
extra={"max_seqlen_q": 4096, **extra},
)
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
@pytest.mark.parametrize("layout", [None, "BSND", "BNSD"])
def test_exact_runtime_layout_scale_and_rotation(runtime, method, layout):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
if layout == "BNSD":
q = q.transpose(1, 2)
impl = flash(layout=layout)
metadata = AttentionMetadata(extra={"kv_cache_dtype": method})
assert impl.forward_npu(q, q, q, metadata) is q
args, kwargs = runtime.quant_attention.call_args
assert args[0] is q # no unconditional transpose or extra quantization
assert kwargs["layout"] == (layout or "BSND") and kwargs["scale"] == 0.37 and kwargs["precision"] == method
assert "attn_mask" not in kwargs
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
assert kwargs["q_rot"] is kwargs["k_rot"]
torch.testing.assert_close(kwargs["q_rot"], get_quant_attention_rotation(q.device, q.dtype, 64))
def test_missing_runtime_requires_config_change(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
del runtime.quant_attention
with pytest.raises(ImportError, match="diffusion_kv_cache_dtype"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
def test_operator_failure_never_falls_back(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
runtime.quant_attention.side_effect = RuntimeError("operator failure")
with pytest.raises(RuntimeError, match="operator failure"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
runtime.quant_attention.assert_called_once()
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
def test_unsupported_shape_requires_config_change(runtime, method):
q = torch.randn(1, 128, 2, 96, dtype=torch.bfloat16)
with pytest.raises(ValueError, match="diffusion_kv_cache_dtype"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": method}))
runtime.quant_attention.assert_not_called()
def test_packed_metadata_never_reaches_quant_runtime(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
metadata = AttentionMetadata(extra={"kv_cache_dtype": "mxfp8", "cu_seqlens_q": torch.tensor([0, 64, 128])})
with pytest.raises(ValueError, match="packed/varlen"):
flash().forward_npu(q, q, q, metadata)
runtime.quant_attention.assert_not_called()
def make_layer(runtime, monkeypatch, method="mxfp8", skip_steps=_DEFAULT_SKIP_STEPS):
layer = object.__new__(layer_mod.Attention)
torch.nn.Module.__init__(layer)
layer.attention = flash()
layer.attn_backend = flash_attn.FlashAttentionBackend
layer._disable_kv_quant = False
layer.layer_idx = 3
cfg = SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices=skip_steps,
diffusion_kv_cache_skip_layer_indices=None,
parallel_config=SimpleNamespace(ring_degree=1),
)
layer._init_kv_cache_quantization(cfg)
return layer
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
def test_global_quant_method_is_published_per_forward(runtime, monkeypatch, method):
layer = make_layer(runtime, monkeypatch, method=method, skip_steps=None)
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == method
@pytest.mark.parametrize("method", [None, "auto", "float"])
def test_optout_without_quant_policy_keeps_metadata_unchanged(runtime, monkeypatch, method):
layer = make_layer(runtime, monkeypatch)
layer._disable_kv_quant = True
layer._init_kv_cache_quantization(
SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices=None,
diffusion_kv_cache_skip_layer_indices=None,
parallel_config=SimpleNamespace(ring_degree=1),
)
)
source = video_metadata()
assert layer._with_kv_cache_dtype(source) is source
def test_step_skip_is_request_local_and_cross_optout(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
ctx = SimpleNamespace(denoise_step_idx=0)
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
source = video_metadata()
for step in list(range(40)) * 2:
ctx.denoise_step_idx = step
resolved = layer._with_kv_cache_dtype(source)
assert resolved.extra["kv_cache_dtype"] == ("float" if step in (0, 1, 38, 39) else "mxfp8")
assert resolved.video_layout is source.video_layout
assert source.extra == {"max_seqlen_q": 4096}
layer._disable_kv_quant = True
assert layer._with_kv_cache_dtype(source).extra["kv_cache_dtype"] == "float"
def test_ring_attention_is_rejected(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
cfg = SimpleNamespace(diffusion_kv_cache_dtype="mxfp8", parallel_config=SimpleNamespace(ring_degree=2))
with pytest.raises(ValueError, match="ring"):
layer._init_kv_cache_quantization(cfg)
def test_sparse_tail_steps_and_dense_quant_dispatch(runtime, monkeypatch):
ctx = SimpleNamespace(denoise_step_idx=49, total_denoise_steps=50)
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: True)
monkeypatch.setattr(rainfusion_attn, "get_forward_context", lambda: ctx)
impl = sparse(start_step=2, end_step=2)
metadata = video_metadata(kv_cache_dtype="fp8")
assert impl._resolve_plan(metadata) is None
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
impl.forward_npu(q, q, q, metadata)
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
def test_sparse_quant_uses_public_runtime(runtime, monkeypatch, method):
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
q = torch.randn(1, 4224, 2, 64, dtype=torch.bfloat16)
out = sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
# The sparse path reuses the query tail as the output padding instead of
# zero-filling a fresh buffer; rows past the 4096-row valid prefix are
# never read back downstream (the model unpads by cu_seqlens), so pin the
# passthrough rather than requiring zeros.
assert out.shape == q.shape and torch.equal(out[:, 4096:], q[:, 4096:])
kwargs = runtime.sparse_attention.call_args.kwargs
assert (kwargs["precision"], kwargs["sparse_type"], kwargs["inner_precise"]) == (method, "rf_v3", 4)
runtime.quant_attention.assert_not_called()
def test_sparse_custom_mask_stays_dense(runtime):
metadata = AttentionMetadata(
attn_mask=torch.ones(1, 4096, dtype=torch.bool),
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32)),
extra={"max_seqlen_q": 4096},
)
assert sparse()._resolve_plan(metadata) is None
def test_sparse_model_padding_mask_is_equivalent_to_trimming(runtime):
metadata = video_metadata()
metadata.attn_mask = torch.arange(4224)[None] < 4096
assert sparse()._resolve_plan(metadata).used_len == 4096
def test_bsa_operator_failure_never_retries(runtime, monkeypatch):
runtime.sparse_attention = Mock(side_effect=RuntimeError("BSA operator failure"))
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="BSA operator failure"):
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype="mxfp4"))
runtime.sparse_attention.assert_called_once()
@pytest.mark.parametrize(
"method,shape,supports_precision,error",
[
("mxfp8", (1, 4096, 2, 64), True, "sparse mxfp8 is not supported"),
("mxfp4", (1, 4096, 2, 64), False, "explicitly support precision"),
("fp8", (2, 4096, 2, 64), True, "batch size 1"),
("fp8", (1, 4096, 2, 32), True, "power-of-two head dimension"),
("mxfp4", (1, 4096, 2, 96), True, "power-of-two head dimension"),
],
)
def test_invalid_sparse_quant_requires_config_change(runtime, monkeypatch, method, shape, supports_precision, error):
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: supports_precision)
q = torch.randn(shape, dtype=torch.bfloat16)
with pytest.raises(ValueError, match=error):
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
runtime.sparse_attention.assert_not_called()
def test_layer_selector_requires_index_except_cross_optout(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
layer.layer_idx = None
config = SimpleNamespace(
diffusion_kv_cache_dtype="mxfp8",
diffusion_kv_cache_skip_step_indices=None,
diffusion_kv_cache_skip_layer_indices={0, 3},
parallel_config=SimpleNamespace(ring_degree=1),
)
with pytest.raises(ValueError, match="parseable transformer block index"):
layer._init_kv_cache_quantization(config)
layer._disable_kv_quant = True
layer._init_kv_cache_quantization(config)
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == "float"
@pytest.mark.parametrize("prefix", ["transformer.blocks.3.attn1", "transformer_2.blocks.3.attn1"])
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
def test_expert_layer_and_step_skip_remain_sparse(runtime, monkeypatch, prefix, method):
def execute(q, k, v, *, precision="bf16", **kwargs):
return q
runtime.sparse_attention = Mock(wraps=execute)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
layer = make_layer(runtime, monkeypatch)
layer.layer_idx = rainfusion_attn._try_extract_layer_index(prefix)
assert layer.layer_idx == 3
config = SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices={0, 1, 38, 39},
diffusion_kv_cache_skip_layer_indices={3},
parallel_config=SimpleNamespace(ring_degree=1),
)
impl = sparse()
layer.attention = impl
layer.attn_backend = rainfusion_attn.RainFusionAttentionBackend
layer._init_kv_cache_quantization(config)
ctx = SimpleNamespace(denoise_step_idx=0)
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
source = video_metadata(kv_cache_dtype="mxfp8")
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
for layer_idx, step in ((3, 0), (3, 2), (4, 0), (4, 1), (4, 2), (4, 37), (4, 38), (4, 39), (4, 0)):
expected = "bf16" if layer_idx == 3 or step in (0, 1, 38, 39) else method
layer.layer_idx = layer_idx
ctx.denoise_step_idx = step
metadata = layer._with_kv_cache_dtype(source)
impl.forward_npu(q, q, q, metadata)
assert runtime.sparse_attention.call_args.kwargs["precision"] == expected
runtime.quant_attention.assert_not_called()
assert source.extra["kv_cache_dtype"] == "mxfp8"
def test_custom_attention_initialization_keeps_upstream_optout(monkeypatch):
monkeypatch.setattr(layer_mod, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kw: object())
monkeypatch.setattr(layer_mod, "NoParallelAttention", lambda: object())
custom = torch.nn.Identity()
layer = layer_mod.Attention(
num_heads=2,
head_size=64,
softmax_scale=0.125,
causal=False,
custom_attention=custom,
skip_sequence_parallel=True,
)
assert layer.attention is custom and layer._kv_cache_dtype is None
def test_legacy_global_fp8_uses_implicit_bsnd(runtime):
q = torch.randn(1, 17, 2, 64, dtype=torch.bfloat16)
impl = flash(layout=None)
assert impl.forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "fp8"})) is q
assert runtime.quant_attention.call_args.kwargs["layout"] == "BSND"
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
@@ -133,18 +133,17 @@ def test_invalid_ref2va_video_spans_fall_back_to_dense():
assert make_impl()._resolve_plan(AttentionMetadata(extra={"max_seqlen_q": 12000}, video_layout=layout)) is None
def test_validate_available_rejects_legacy_mindiesd(monkeypatch):
def test_validate_available_accepts_single_video_mindiesd(monkeypatch):
mindiesd = types.ModuleType("mindiesd")
def sparse_attention(query, key, value, **kwargs):
return query
mindiesd.sparse_attention = sparse_attention
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
with pytest.raises(ValueError, match="video_spans"):
RainFusionAttentionBackend.validate_available()
RainFusionAttentionBackend.validate_available()
def test_validate_available_accepts_new_mindiesd(monkeypatch):
@@ -153,13 +152,25 @@ def test_validate_available_accepts_new_mindiesd(monkeypatch):
def sparse_attention(query, key, value, *, video_spans=None, **kwargs):
return query
mindiesd.sparse_attention = sparse_attention
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
RainFusionAttentionBackend.validate_available()
def test_prefix_kv_slicing_contract_is_pinned():
# Regression pin for the #5543 packed-padding crash: reverting the flag
# would pass CI while reintroducing the ValueError for non-64-aligned
# requests. Both halves of the contract must hold together — the impl
# trims [real, pad] packed tensors to the valid prefix itself (so it may
# advertise prefix slicing) and it never reads attn_mask (so it must keep
# refusing one), which is what lets the model skip materializing the
# padding mask that _assert_metadata_compatible would reject.
assert RainFusionAttentionBackend.supports_prefix_kv_slicing is True
assert RainFusionAttentionBackend.supports_attention_mask() is False
@pytest.mark.parametrize("grid", [(4, 24, 40), (1, 24, 40)])
def test_short_video_stays_dense(grid):
assert make_impl()._resolve_plan(make_metadata(grid)) is None
@@ -300,22 +311,21 @@ def _fake_mindiesd_module():
import types
fake = types.ModuleType("mindiesd")
fake.sparse_attention = lambda *args, **kwargs: None
setattr(fake, "sparse_attention", lambda *args, **kwargs: None)
return fake
def test_precision_non_bf16_requires_mindiesd_support():
"""precision != bf16 against a mindiesd lacking the kwarg must raise RuntimeError."""
"""Unsupported sparse precision must fail before any operator executes."""
import sys
impl = make_impl(precision="mix")
sys.modules["mindiesd"] = _fake_mindiesd_module()
try:
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=False):
with pytest.raises(RuntimeError, match="requires MindIE-SD"):
impl._forward_sparse_npu(None, None, None, None)
finally:
sys.modules.pop("mindiesd", None)
with pytest.raises(ValueError, match="explicitly support precision"):
impl._forward_sparse_npu(
None, None, None, RainFusionPlan(prefix_len=0, used_len=8, latent_shape=[2, 2, 2])
)
def test_precision_non_bf16_passes_gate_when_supported():
@@ -323,8 +333,7 @@ def test_precision_non_bf16_passes_gate_when_supported():
import sys
impl = make_impl(precision="mix")
sys.modules["mindiesd"] = _fake_mindiesd_module()
try:
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=True):
# q/k/v shapes: [B, S, N, D]; plan geometry must match S.
q = torch.randn(1, 8, 4, 128)
@@ -332,5 +341,3 @@ def test_precision_non_bf16_passes_gate_when_supported():
# The gate must pass; the fake mindiesd returns None so no crash.
out = impl._forward_sparse_npu(q, q, q, plan)
assert out is None
finally:
sys.modules.pop("mindiesd", None)
@@ -325,7 +325,7 @@ def _runner(
model_config=SimpleNamespace(max_model_len=max_len),
scheduler_config=SimpleNamespace(
max_num_batched_tokens=max_num_batched_tokens,
max_num_seqs=max_num_seqs,
max_num_seqs=max_num_seqs * max_rows_per_request,
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=cp_size,
@@ -403,7 +403,7 @@ def test_initialize_failure_does_not_publish_partial_cache_state(
successful_init_cache = model_runner_backend_module.init_kv_cache
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=16),
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=1),
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=2),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
cp_kv_cache_interleave_size=1,
@@ -445,7 +445,7 @@ def test_initialize_failure_does_not_publish_partial_cache_state(
assert backend.block_tables is None
assert backend.paged_attention_adapter is None
assert backend._kv_cache_layer_adapters["layer-0"].kv_cache == "placeholder-0"
assert backend.vllm_config.scheduler_config.max_num_seqs == 1
assert backend.vllm_config.scheduler_config.max_num_seqs == 2
monkeypatch.setattr(model_runner_backend_module, "init_kv_cache", successful_init_cache)
backend.initialize_kv_cache(config)
@@ -58,6 +58,7 @@ def _od_config(**overrides):
gpu_memory_utilization=0.9,
kv_cache_memory_bytes=None,
max_num_seqs=1,
diffusion_kv_max_rows_per_request=1,
max_num_batched_tokens=64,
num_gpus=1,
)
@@ -1,16 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from contextlib import nullcontext
from types import SimpleNamespace
from unittest import mock
import pytest
import torch
from vllm.config import KVTransferConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.outputs import KVConnectorOutput
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.diffusion_kv.kv_connector import (
KVTransferRegistrationError,
commit_kv_load,
parse_kv_transfer_config,
shutdown_kv_connector,
wait_for_kv_load,
)
from vllm_omni.diffusion.sched.base_scheduler import BaseScheduler
from vllm_omni.diffusion.vllm_config import create_diffusion_vllm_config
@@ -138,3 +145,205 @@ def test_shutdown_kv_connector_is_idempotent() -> None:
assert ensure_shutdown.call_count == 2
scheduler_connector.shutdown.assert_called_once()
def test_commit_aligns_cfg_transport_pages_without_extending_computed_prefix() -> None:
requests = tuple(
SimpleNamespace(
request_id=f"req/{sequence_id}",
kv_transfer_params={"num_transfer_tokens": 9},
num_tokens=16,
num_computed_tokens=0,
)
for sequence_id in range(2)
)
manager = SimpleNamespace(
get_blocks=lambda _request_id: KVCacheBlocks(([0, 1, 2, 3],)),
kv_cache_config=SimpleNamespace(kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=4))]),
)
connector = mock.Mock()
expected = commit_kv_load(connector, manager, requests, [7, 5])
assert expected == {"req/0", "req/1"}
calls = connector.update_state_after_alloc.call_args_list
assert [len(call.args[1].blocks[0]) for call in calls] == [3, 3]
assert [call.args[2] for call in calls] == [7, 5]
assert [request.num_computed_tokens for request in requests] == [7, 5]
def test_wait_without_pending_load_uses_vllm_post_forward_once() -> None:
connector = mock.Mock()
output = mock.Mock(invalid_block_ids=set())
active_connector = mock.Mock(kv_connector=connector)
active_connector.post_forward.return_value = output
scheduler_output = SimpleNamespace(
kv_transfer_request_ids=set(),
finished_req_ids={"finished"},
kv_finished_request_ids={"finished/diffusion-kv/0", "finished/diffusion-kv/1"},
)
result = wait_for_kv_load(active_connector, scheduler_output, timeout=1.0)
assert result is output
active_connector.pre_forward.assert_called_once_with(scheduler_output)
active_connector.post_forward.assert_called_once_with({"finished/diffusion-kv/0", "finished/diffusion-kv/1"})
connector.get_finished.assert_not_called()
def test_timeout_returns_partial_completion_without_releasing_pages():
active = mock.Mock()
active.kv_connector.get_finished.return_value = (set(), {"cfg0"})
active.kv_connector.get_block_ids_with_load_errors.return_value = set()
active.post_forward.return_value = KVConnectorOutput()
scheduled = SimpleNamespace(kv_transfer_request_ids={"cfg0", "cfg1"}, kv_finished_request_ids=set())
output = wait_for_kv_load(active, scheduled, timeout=0)
assert output.finished_recving == {"cfg0"}
active.kv_connector.request_finished.assert_not_called()
active.post_forward.assert_called_once_with(set())
@pytest.fixture
def cfg_registration():
requests = tuple(
SimpleNamespace(
request_id=f"cfg{i}",
status=None,
num_tokens=8,
kv_transfer_params={"num_transfer_tokens": 4, "transfer_id": "ticket", "do_remote_prefill": True},
)
for i in range(2)
)
manager = SimpleNamespace(
get_blocks=lambda _: KVCacheBlocks(([0, 1],)),
kv_cache_config=SimpleNamespace(kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=4))]),
)
return requests, manager
def test_cfg_validation_is_atomic_before_connector_registration(cfg_registration):
requests, manager = cfg_registration
requests[1].kv_transfer_params["num_transfer_tokens"] = 9
connector = mock.Mock()
with pytest.raises(KVTransferRegistrationError, match="boundary"):
commit_kv_load(connector, manager, requests, [4, 4])
connector.update_state_after_alloc.assert_not_called()
@pytest.mark.parametrize("fail_init", [False, True])
def test_sp_connector_init_snapshots_topology_and_restores_model_tp(monkeypatch, fail_init):
import vllm.distributed.parallel_state as parallel_state
import vllm_omni.diffusion.diffusion_kv.kv_connector as module
import vllm_omni.diffusion.distributed.parallel_state as omni_parallel
tp = SimpleNamespace(world_size=1, rank_in_group=0)
sp = SimpleNamespace(world_size=2, rank_in_group=1)
monkeypatch.setattr(parallel_state, "_TP", tp)
monkeypatch.setattr(omni_parallel, "get_sp_group", lambda: sp)
snapshot: dict[str, int] = {}
def initialize(*args):
snapshot.update(
rank=parallel_state.get_tensor_model_parallel_rank(),
size=parallel_state.get_tensor_model_parallel_world_size(),
)
if fail_init:
raise RuntimeError("initialization failed")
monkeypatch.setattr(module, "ensure_kv_transfer_initialized", initialize)
config = SimpleNamespace(
kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector", engine_id="dit"),
parallel_config=SimpleNamespace(prefill_context_parallel_size=2),
)
with pytest.raises(RuntimeError, match="initialization failed") if fail_init else nullcontext():
module.init_worker_kv_connector(config, object())
assert parallel_state._TP is tp
assert snapshot == {"rank": 1, "size": 2}
def test_partial_mooncake_registration_is_replaced_by_empty_notifications(cfg_registration):
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
MooncakeConnector,
MooncakeConnectorScheduler,
)
scheduler = object.__new__(MooncakeConnectorScheduler)
scheduler.is_kv_producer = False
scheduler.is_kv_consumer = True
scheduler._reqs_need_recv = {}
connector = object.__new__(MooncakeConnector)
connector.connector_scheduler = scheduler
requests, manager = cfg_registration
for request in requests:
request.kv_transfer_params.update(remote_engine_id="ar", remote_bootstrap_addr="http://localhost:8998")
def register(request, blocks, count):
scheduler._reqs_need_recv[request.request_id] = (request, blocks)
request.kv_transfer_params["do_remote_prefill"] = False
if request.request_id == "cfg1":
raise RuntimeError("partial registration")
connector.update_state_after_alloc = register
with pytest.raises(KVTransferRegistrationError, match="partial registration"):
commit_kv_load(connector, manager, requests, [4, 4])
assert set(scheduler._reqs_need_recv) == {"cfg0", "cfg1"}
assert all(blocks == [] for _, blocks in scheduler._reqs_need_recv.values())
metadata = scheduler.build_connector_meta(None)
# Even partial registration failure sends every row in one per-rank batch,
# preserving the producer's complete fan-out count for the shared ticket.
assert set(metadata.reqs_to_recv["ar"]) == {"cfg0", "cfg1"}
assert all(
meta.transfer_id == "ticket" and meta.local_block_ids == [] for meta in metadata.reqs_to_recv["ar"].values()
)
@pytest.mark.parametrize("timeout", [0, -1, float("inf"), "60"])
def test_native_transfer_timeout_must_be_positive_finite(timeout):
with pytest.raises(ValueError, match="transfer_timeout"):
parse_kv_transfer_config(
{
**KV_TRANSFER_CONFIG,
"kv_connector_extra_config": {"transfer_timeout": timeout},
}
)
def test_receive_completion_requires_all_ranks_across_polls():
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
from vllm_omni.diffusion.sched.interface import CachedRequestData, DiffusionSchedulerOutput
executor = SimpleNamespace(od_config=SimpleNamespace(num_gpus=2))
executor.collective_rpc = mock.Mock(
side_effect=[
[KVConnectorOutput(finished_recving={"cfg0", "cfg1"}), KVConnectorOutput(finished_recving={"cfg0"})],
[KVConnectorOutput(), KVConnectorOutput(finished_recving={"cfg1"})],
]
)
scheduled = DiffusionSchedulerOutput(
step_id=0,
scheduled_new_reqs=[],
scheduled_cached_reqs=CachedRequestData.make_empty(),
finished_req_ids=set(),
num_running_reqs=0,
num_waiting_reqs=0,
kv_connector_metadata=object(),
kv_transfer_request_ids={"cfg0", "cfg1"},
)
result = DiffusionExecutor.prepare_kv_for_forward(executor, scheduled)
assert result.finished_recving == {"cfg0"}
scheduled.kv_transfer_request_ids = set()
result = DiffusionExecutor.prepare_kv_for_forward(executor, scheduled)
assert result.finished_recving == {"cfg1"}
assert not executor._kv_receive_completed_ranks
def test_native_transfer_rejects_sleep_without_changing_legacy_sleep():
with pytest.raises(ValueError, match="registered pages must remain mapped"):
OmniDiffusionConfig.from_kwargs(
diffusion_kv_mode="paged_scheduler",
diffusion_kv_max_rows_per_request=2,
kv_transfer_config=dict(KV_TRANSFER_CONFIG),
enable_sleep_mode=True,
)
assert OmniDiffusionConfig.from_kwargs(enable_sleep_mode=True).enable_sleep_mode
@@ -61,13 +61,14 @@ class _MetadataScheduler(BaseScheduler):
def test_scheduler_copies_opaque_params_to_sequence_requests() -> None:
params = {"transfer_id": "xfer-req-0", "do_remote_prefill": True}
params = {"transfer_id": "xfer-req-0", "do_remote_prefill": True, "num_transfer_tokens": 3}
sequence = DiffusionKVRequest(
"req-0/diffusion-kv/0",
sequence_id=0,
prefix_len=4,
target_len=2,
seq_len=6,
prompt_token_ids=[10, 11, 12, 13],
)
request = OmniDiffusionRequest(
prompt="prompt",
@@ -83,6 +84,9 @@ def test_scheduler_copies_opaque_params_to_sequence_requests() -> None:
assert state.diffusion_kv_requests[0].kv_transfer_params == params
assert state.diffusion_kv_requests[0].kv_transfer_params is not params
assert sequence.prompt_token_ids == [10, 11, 12]
assert sequence.num_prompt_tokens == 3
assert sequence.seq_len == 6
def test_diffusion_kv_metadata_uses_native_cache_group_block_ids() -> None:
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Exercise upstream Mooncake completion accounting without device/network I/O."""
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import msgspec
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector as mc
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.outputs import KVConnectorOutput
from vllm_omni.diffusion.diffusion_kv.kv_connector import install_mooncake_cfg_fanout
from vllm_omni.worker.mixins import OmniWorkerMixin
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@pytest.fixture
def producer(monkeypatch):
worker = object.__new__(mc.MooncakeConnectorWorker)
worker.shutdown = Mock() # No hardware constructor, sockets or listener thread.
worker.is_kv_consumer = False
worker.tp_rank, worker.tp_size = 0, 1
worker.transfer_topo = SimpleNamespace(handshake_target_ranks=lambda size: list(range(size)))
worker.reqs_need_send, worker.finished_sending_reqs = {}, set()
for name in (
"kv_caches_base_addr",
"block_len_per_layer",
"kv_block_len_per_layer",
"registered_layer_names",
"registered_layer_indices",
"registered_group_indices",
):
setattr(worker, name, [])
worker._get_transfer_regions = Mock(return_value=[])
worker._producer_cache_is_replicated = Mock(return_value=False)
worker._encoder = msgspec.msgpack.Encoder()
worker._sender_executor = None
worker._send_blocks = Mock(return_value=0)
worker.xfer_stats = Mock()
worker._build_transfer_params = AsyncMock(return_value=([1], [2], [16], [], None))
monkeypatch.setattr(mc, "_align_transfer_regions", lambda *_: ([], [], None))
monkeypatch.setattr(mc, "_validate_asymmetric_region_lengths", lambda **_: None)
monkeypatch.setattr(mc.envs, "VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", 0.05)
connector = object.__new__(mc.MooncakeConnector)
connector.connector_worker = worker
return connector
async def register(worker, ticket):
worker.sender_loop = asyncio.get_running_loop()
for blocks in ([], [[10]]):
meta = mc.MooncakeConnectorMetadata()
meta.add_new_req(ticket, blocks, {"transfer_id": ticket}, load_remote_cache=False)
await worker.record_send_reqs(meta)
return worker.reqs_need_send[ticket]
def packet(rank, ranks, tickets, empty=False):
return mc.MooncakeXferMetadata(
remote_hostname="localhost",
remote_port=1234 + rank,
remote_tp_size=ranks,
remote_tp_rank=rank,
req_blocks={
f"{ticket}/{row}": (ticket, [] if empty else [[100 + rank * 10 + row]])
for ticket, rows in tickets.items()
for row in range(rows)
},
kv_caches_base_addr=[],
block_lens=[],
kv_block_lens=[],
)
async def send(worker, rank, ranks, tickets, empty=False):
sock = SimpleNamespace(send_multipart=AsyncMock())
await worker.send_kv_to_decode(str(rank).encode(), sock, packet(rank, ranks, tickets, empty))
return [
msgspec.msgpack.decode(c.args[0][1], type=mc.MooncakeXferResponse) for c in sock.send_multipart.call_args_list
]
@pytest.mark.asyncio
@pytest.mark.parametrize("rows", [1, 2])
@pytest.mark.parametrize("ranks", [2, 4])
@pytest.mark.parametrize("in_flight", [False, True])
async def test_source_pages_wait_for_every_rank_and_row(producer, rows, ranks, in_flight):
worker = producer.connector_worker
entered, release = asyncio.Event(), asyncio.Event()
original = worker._build_transfer_params
async def plan(ready, meta, *regions):
if in_flight and meta.remote_tp_rank == ranks - 1:
entered.set()
await release.wait()
return await original(ready, meta, *regions)
worker._build_transfer_params = plan
install_mooncake_cfg_fanout(producer)
adapted = worker._build_transfer_params
install_mooncake_cfg_fanout(producer)
assert worker._build_transfer_params is adapted # Do not multiply twice.
ticket = await register(worker, "request")
pending = asyncio.create_task(send(worker, ranks - 1, ranks, {"request": rows})) if in_flight else None
try:
if pending is not None:
await asyncio.wait_for(entered.wait(), 2)
for rank in range(ranks - 1):
await send(worker, rank, ranks, {"request": rows})
assert ticket.need_send == ranks * rows
assert ticket.sent == (rank + 1) * rows
assert worker.reqs_need_send["request"] is ticket
finished = await worker.fetch_finished_sending_reqs()
scheduler = SimpleNamespace(connector=None, requests={"request": object()}, _free_blocks=Mock())
Scheduler._update_from_kv_xfer_finished(scheduler, KVConnectorOutput(finished_sending=finished))
scheduler._free_blocks.assert_not_called()
release.set()
responses = (
await asyncio.wait_for(pending, 2) if pending else await send(worker, ranks - 1, ranks, {"request": rows})
)
assert sum(len(response.ok_reqs or []) for response in responses) == rows
assert all(not response.err_reqs for response in responses)
assert "request" not in worker.reqs_need_send
finished = await worker.fetch_finished_sending_reqs()
assert finished == {"request"}
Scheduler._update_from_kv_xfer_finished(scheduler, KVConnectorOutput(finished_sending=finished))
scheduler._free_blocks.assert_called_once()
assert not await worker.fetch_finished_sending_reqs()
finally:
release.set()
if pending is not None:
await asyncio.gather(pending, return_exceptions=True)
@pytest.mark.asyncio
@pytest.mark.parametrize("empty", [False, True])
@pytest.mark.parametrize("ranks", [1, 2])
async def test_mixed_tickets_and_empty_block_notifications(producer, empty, ranks):
worker = producer.connector_worker
if empty:
worker._build_transfer_params.return_value = ([], [], [], [], None)
install_mooncake_cfg_fanout(producer)
single, cfg = await register(worker, "single"), await register(worker, "cfg")
await asyncio.gather(*(send(worker, rank, ranks, {"single": 1, "cfg": 2}, empty) for rank in range(ranks)))
assert (single.need_send, single.sent) == (ranks, ranks)
assert (cfg.need_send, cfg.sent) == (2 * ranks, 2 * ranks)
assert await worker.fetch_finished_sending_reqs() == {"single", "cfg"}
assert not worker.reqs_need_send
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["row_count", "transport"])
async def test_failed_rank_does_not_report_source_complete(producer, failure):
worker = producer.connector_worker
install_mooncake_cfg_fanout(producer)
ticket = await register(worker, "request")
await send(worker, 0, 2, {"request": 2})
if failure == "transport":
worker._send_blocks.return_value = -1
responses = await send(worker, 1, 2, {"request": 1 if failure == "row_count" else 2})
assert any(response.err_reqs for response in responses)
assert ticket.sent == 2 and ticket.need_send == 4 and ticket.sending == 0
assert worker.reqs_need_send["request"] is ticket
assert not await worker.fetch_finished_sending_reqs()
# Preserve upstream timeout cleanup after all in-flight writes have stopped.
ticket.expire_time = 0
assert await worker.fetch_finished_sending_reqs() == {"request"}
assert not worker.reqs_need_send
def test_worker_installs_adapter_after_native_initialization(producer, monkeypatch):
import vllm.distributed.kv_transfer as kv_transfer
worker = producer.connector_worker
original = worker._build_transfer_params
class UpstreamWorker:
def initialize_from_config(self, config):
assert config == "config"
assert worker._build_transfer_params is original
class OmniWorker(OmniWorkerMixin, UpstreamWorker):
pass
monkeypatch.setattr(kv_transfer, "has_kv_transfer_group", lambda: True)
monkeypatch.setattr(kv_transfer, "get_kv_transfer_group", lambda: producer)
object.__new__(OmniWorker).initialize_from_config("config")
assert worker._build_transfer_params is not original
@pytest.mark.parametrize("role", ["other_connector", "scheduler", "consumer"])
def test_adapter_leaves_other_roles_untouched(producer, role):
worker = producer.connector_worker
original = worker._build_transfer_params
if role == "scheduler":
producer.connector_worker = None
elif role == "consumer":
worker.is_kv_consumer = True
install_mooncake_cfg_fanout(object() if role == "other_connector" else producer)
assert worker._build_transfer_params is original
@@ -94,7 +94,7 @@ class _FakeAttentionGroup:
return self.builder
@dataclass(frozen=True, slots=True)
@dataclass(slots=True)
class _FakeNativeMetadata:
build_id: int
causal: bool
@@ -102,6 +102,7 @@ class _FakeNativeMetadata:
query_start_loc_cpu: torch.Tensor
positions: torch.Tensor
slot_mappings: torch.Tensor
max_num_splits: int = 0
class _FakeLayer:
@@ -288,6 +289,8 @@ def test_omni_paged_backend_consumes_context_and_restores_diffusion_shape(
assert output.shape == query.shape
assert torch.equal(output, query)
assert layer.calls[0][0].shape == (5, 2, 4)
assert layer.calls[0][1] is None
assert layer.calls[0][2] is None
assert layer.native_events == ["update", "forward"]
assert layer.calls[0][3] is events[0][2]
@@ -440,9 +443,7 @@ def test_platform_prewrite_updates_once_then_piecewise_reads_cache(monkeypatch:
def test_omni_paged_backend_runs_hunyuan_piecewise_segments(monkeypatch: pytest.MonkeyPatch) -> None:
adapter, _, layer, events = _make_adapter(monkeypatch)
# Exercise the Ascend-only homogeneous batch path. CUDA keeps its native
# output-buffer contract and is covered by the GPU adapter tests.
monkeypatch.setattr(adapter_module.current_omni_platform, "is_npu", lambda: True)
layer.impl.vllm_flash_attn_version = 2
batch = adapter.prepare_batch(
[
DiffusionPagedAttentionRow(
@@ -499,10 +500,10 @@ def test_omni_paged_backend_runs_hunyuan_piecewise_segments(monkeypatch: pytest.
assert torch.equal(layer.updates[0][0], key.reshape(12, 2, 4))
assert torch.equal(layer.updates[0][1], value.reshape(12, 2, 4))
assert [call[0].shape[0] for call in layer.calls] == [4, 6, 2]
assert [call[1].shape[0] for call in layer.calls] == [4, 6, 2]
assert [call[2].shape[0] for call in layer.calls] == [4, 6, 2]
assert all(call[1] is None and call[2] is None for call in layer.calls)
assert all(call[3] is event[2] for call, event in zip(layer.calls, events[1:], strict=True))
assert [metadata.build_id for metadata in context.piecewise_native_metadata] == [1, 2, 3]
assert [metadata.max_num_splits for metadata in context.piecewise_native_metadata] == [1, 1, 1]
# The first metadata build is the normal whole-query path. Piecewise calls
# then use causal [3, 5), full [5, 8), and causal [8, 9) segments.
@@ -539,6 +540,51 @@ def test_omni_paged_backend_runs_hunyuan_piecewise_segments(monkeypatch: pytest.
]
def test_homogeneous_piecewise_packs_strided_query_only_per_segment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
adapter, _, _, _ = _make_adapter(monkeypatch)
batch = adapter.prepare_batch(
[
DiffusionPagedAttentionRow(
request_id="req-0",
sequence_id=0,
query_len=6,
seq_len=9,
kv_start_pos=3,
),
DiffusionPagedAttentionRow(
request_id="req-1",
sequence_id=1,
query_len=6,
seq_len=9,
kv_start_pos=3,
),
]
)
packed_qkv = torch.randn(2, 6, 3, 2, 4)
query, key, value = packed_qkv.unbind(dim=2)
assert not query.is_contiguous()
metadata = SimpleNamespace(
attn_mask=None,
full_attn_spans=[[(5, 8)], [(5, 8)]],
query_ranges=None,
extra={},
)
with adapter.activate(batch):
context = adapter.prepare_layer_context(
"layer-0",
query,
key,
value,
omni_attn_metadata=metadata,
)
assert not context.query.is_contiguous()
assert context.query.data_ptr() == query.data_ptr()
def test_piecewise_metadata_snapshots_reused_native_scheduler_buffer(monkeypatch: pytest.MonkeyPatch) -> None:
adapter, _, _, _ = _make_adapter(monkeypatch)
shared_scheduler_metadata = torch.zeros(1, dtype=torch.int32)
@@ -1232,7 +1278,7 @@ def test_layer_adapter_accepts_platform_native_backend_and_uses_rank_local_heads
ulysses_degree=2,
)
assert selected_backends == [(adapter_module.AttentionBackendEnum.FLASH_ATTN, {}, 2)]
assert selected_backends == [(adapter_module.AttentionBackendEnum.FLASH_ATTN, {}, 1)]
assert specialized_backends == [(native_backend, 2)]
assert config.attention_config.backend is None
assert config.attention_config.backend_per_kind is original_backend_per_kind
@@ -43,8 +43,7 @@ class _SmokeDiffusionAttention(nn.Module):
self.softmax_scale = _HEAD_SIZE**-0.5
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for native paged attention")
def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> None:
def _native_adapter(num_rows: int = 1):
device = torch.device("cuda", torch.accelerator.current_device_index())
vllm_config = VllmConfig(
cache_config=CacheConfig(
@@ -89,8 +88,8 @@ def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> N
canonical_spec = native_layer.spec
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[build_kv_cache_tensor(canonical_spec, 4, [_LAYER_NAME])],
num_blocks=4 * num_rows,
kv_cache_tensors=[build_kv_cache_tensor(canonical_spec, 4 * num_rows, [_LAYER_NAME])],
kv_cache_groups=[
KVCacheGroupSpec(
layer_names=[_LAYER_NAME],
@@ -120,14 +119,15 @@ def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> N
)
block_tables = BlockTables(
block_sizes=[_BLOCK_SIZE],
max_num_reqs=1,
max_num_batched_tokens=32,
max_num_reqs=num_rows,
max_num_batched_tokens=32 * num_rows,
max_num_blocks_per_group=[2],
device=device,
kernel_block_sizes=kernel_block_sizes,
)
block_tables.append_block_ids(0, ([3, 1],), overwrite=True)
for row in range(num_rows):
block_tables.append_block_ids(row, ([4 * row + 3, 4 * row + 1],), overwrite=True)
block_tables.apply_staged_writes()
adapter = DiffusionPagedAttentionAdapter(
vllm_config=vllm_config,
@@ -136,11 +136,17 @@ def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> N
block_tables=block_tables,
attn_groups=attn_groups,
layers={_LAYER_NAME: native_layer},
resolve_row=lambda _request_id, _sequence_id, _context_id: DiffusionPagedAttentionRowBinding(
row_index=0,
resolve_row=lambda _request_id, sequence_id, _context_id: DiffusionPagedAttentionRowBinding(
row_index=sequence_id,
max_seq_len=19,
),
)
return adapter, diffusion_layer, device
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for native paged attention")
def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> None:
adapter, diffusion_layer, device = _native_adapter()
prefix_batch = adapter.prepare_batch(
[
DiffusionPagedAttentionRow(
@@ -258,3 +264,48 @@ def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> N
torch.testing.assert_close(prefix_output, prefix_reference, rtol=2e-2, atol=2e-2)
torch.testing.assert_close(suffix_output, suffix_reference, rtol=2e-2, atol=2e-2)
torch.testing.assert_close(piecewise_output, piecewise_reference, rtol=2e-2, atol=2e-2)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for native paged attention")
def test_hunyuan_homogeneous_cfg_paged_attention_graph_replay() -> None:
"""Real paged kernels, strided QKV, mixed image spans and changed replay inputs."""
adapter, layer, device = _native_adapter(num_rows=2)
packed = torch.randn(2, 19, 3, _NUM_HEADS, _HEAD_SIZE, device=device, dtype=torch.float16)
query, key, value = packed.unbind(dim=2)
backend = FlashAttentionImpl(
num_heads=_NUM_HEADS, head_size=_HEAD_SIZE, softmax_scale=layer.softmax_scale, num_kv_heads=_NUM_HEADS
)
batch = adapter.prepare_batch(
[DiffusionPagedAttentionRow(request_id="cfg", sequence_id=i, query_len=19, seq_len=19) for i in range(2)]
)
metadata = AttentionMetadata(full_attn_spans=[[(5, 10)], [(5, 10)]])
with adapter.activate(batch):
context = adapter.prepare_layer_context(_LAYER_NAME, query, key, value, omni_attn_metadata=metadata)
assert context.piecewise_plan.homogeneous_batch_shape == (2, 19)
warmup = torch.cuda.Stream()
warmup.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(warmup):
for _ in range(3):
backend.forward_paged(context)
torch.cuda.current_stream().wait_stream(warmup)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = backend.forward_paged(context)
output_ptr = captured.data_ptr()
mask = torch.ones(19, 19, device=device, dtype=torch.bool).tril()
mask[5:10, 5:10] = True
for _ in range(2):
packed.copy_(torch.randn_like(packed))
graph.replay()
actual = captured.clone()
eager = backend.forward_paged(context)
reference = F.scaled_dot_product_attention(
query.transpose(1, 2),
key.transpose(1, 2),
value.transpose(1, 2),
attn_mask=mask,
scale=layer.softmax_scale,
).transpose(1, 2)
torch.testing.assert_close(actual.reshape_as(reference), reference, rtol=2e-2, atol=2e-2)
torch.testing.assert_close(actual, eager, rtol=2e-2, atol=2e-2)
assert captured.data_ptr() == output_ptr
@@ -176,10 +176,11 @@ def test_worker_honors_explicit_kv_memory_budget(monkeypatch) -> None:
profile_request = object()
assert worker.determine_available_kv_memory(profile_request) == [4096]
worker.model_runner.profile_run.assert_called_once_with(profile_request)
worker.model_runner.profile_run.assert_not_called()
def test_worker_treats_zero_kv_memory_budget_as_profiled_auto_sizing(monkeypatch) -> None:
monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", SimpleNamespace(empty_cache=Mock()))
worker = object.__new__(DiffusionWorker)
worker.rank = 0
worker.init_snapshot = SimpleNamespace(free_memory=900)
@@ -209,6 +210,7 @@ def test_worker_treats_zero_kv_memory_budget_as_profiled_auto_sizing(monkeypatch
@pytest.mark.parametrize("non_kv_cache_memory", [750, 800])
def test_worker_rejects_non_positive_profiled_kv_memory(monkeypatch, non_kv_cache_memory: int) -> None:
monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", SimpleNamespace(empty_cache=Mock()))
worker = object.__new__(DiffusionWorker)
worker.rank = 0
worker.init_snapshot = SimpleNamespace(free_memory=900)
@@ -245,6 +247,7 @@ def test_worker_rejects_non_positive_profiled_kv_memory(monkeypatch, non_kv_cach
def test_worker_profiles_activation_headroom_instead_of_current_residency(monkeypatch) -> None:
monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", SimpleNamespace(empty_cache=Mock()))
worker = object.__new__(DiffusionWorker)
worker.rank = 0
worker.init_snapshot = SimpleNamespace(free_memory=1200)
@@ -310,3 +313,31 @@ def test_rank_probe_gathers_local_failure_before_raising(monkeypatch) -> None:
)
assert gathered == [(False, "ValueError: local failure")]
def test_hunyuan_native_layer_identity_matches_ar_spec(monkeypatch):
import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_image3_transformer as hy3
monkeypatch.setattr(hy3, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(hy3, "get_sequence_parallel_world_size", lambda: 1)
monkeypatch.setattr(hy3, "get_allgather_parallel_world_size", lambda: 1)
monkeypatch.setattr(hy3, "get_sequence_parallel_rank", lambda: 0)
monkeypatch.setattr(hy3, "QKVParallelLinear", lambda **_: nn.Identity())
monkeypatch.setattr(hy3, "RowParallelLinear", lambda **_: nn.Identity())
monkeypatch.setattr(hy3, "get_rope", lambda **_: nn.Identity())
# Only backend construction is fake; execute both HY3 constructors and the
# real runner spec-discovery path, including the actual Attention prefix.
def attention(**kwargs):
return _attention(enabled=kwargs.get("paged_kv_cache_role") is not None, prefix=kwargs["prefix"])
monkeypatch.setattr(hy3, "Attention", attention)
model = hy3.HunYuanAttention(
config=SimpleNamespace(num_key_value_heads=2, attention_head_dim=8),
hidden_size=16,
num_heads=2,
num_kv_heads=2,
prefix="layers.3.self_attn",
)
runner = _runner(model.image_attn.attn)
assert set(runner.get_kv_cache_spec()) == {"model.layers.3.self_attn.attn"}
+43
View File
@@ -461,3 +461,46 @@ def test_ring_p2p_parity(world_size: int, dtype: torch.dtype):
device_kind="cuda",
master_port=29501,
)
def _run_fused_qkv_matches_three_4d(
local_rank: int,
world_size: int,
device_kind: DeviceKind,
master_port: int,
) -> None:
"""LingBot's fused (B, S/N, 3, H, D) exchange must equal three per-tensor 4D exchanges."""
from vllm_omni.diffusion.distributed.comm import all_to_all_5D
device = _worker_device(local_rank, device_kind)
if device_kind == "cuda":
current_omni_platform.set_device(device)
_init_worker(local_rank, world_size, master_port, device_kind)
initialize_model_parallel(ulysses_degree=world_size)
sp_group = get_sp_group().ulysses_group
try:
batch, seq_per_rank, heads, head_size = 1, 6, 4 * world_size, 8
# Distinguishable values: rank, q/k/v index, position, head, feature.
base = torch.arange(seq_per_rank * heads * head_size, dtype=torch.float32, device=device).reshape(
batch, seq_per_rank, heads, head_size
)
tensors = [base + 1000.0 * local_rank + 100000.0 * which for which in range(3)]
separate = [SeqAllToAll4D.apply(sp_group, t, 2, 1, False) for t in tensors]
fused = all_to_all_5D(torch.stack(tensors, dim=2), scatter_idx=3, gather_idx=1, group=sp_group)
assert fused.shape == (batch, seq_per_rank * world_size, 3, heads // world_size, head_size)
for which, expected in enumerate(separate):
assert torch.equal(fused.unbind(2)[which], expected), f"q/k/v index {which} differs on rank {local_rank}"
finally:
destroy_distributed_env()
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.cpu
@pytest.mark.parametrize("world_size", [2, 4])
def test_fused_qkv_all_to_all_matches_three_4d_exchanges_cpu(world_size: int) -> None:
torch.multiprocessing.spawn(
_run_fused_qkv_matches_three_4d,
args=(world_size, "cpu", 29613),
nprocs=world_size,
)
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for SP subgroup construction in parallel_state.py."""
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for SP subgroup construction for diffusion parallel state."""
from __future__ import annotations
@@ -8,12 +8,55 @@ from types import SimpleNamespace
import pytest
import torch
import vllm.distributed.parallel_state as vllm_parallel_state
from tests.helpers.runtime import get_distributed_init_method
from vllm_omni.diffusion.distributed import parallel_state as omni_parallel_state
from vllm_omni.diffusion.distributed.parallel_state import RankGenerator, set_seq_parallel_pg
pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.core_model, pytest.mark.cpu]
def ensure_parallel_state_initialized():
"""Ensure torch distributed is initialized and that all world / count vars are set."""
assert torch.distributed.is_initialized()
assert omni_parallel_state._WORLD is not None
assert vllm_parallel_state._WORLD is not None
assert vllm_parallel_state._NODE_COUNT is not None
def ensure_parallel_state_not_initialized():
"""Ensure torch distributed is not initialized and that all world / count vars are unset."""
assert not torch.distributed.is_initialized()
assert omni_parallel_state._WORLD is None
assert vllm_parallel_state._WORLD is None
assert vllm_parallel_state._NODE_COUNT is None
def test_omni_manages_vllm_distributed_state(monkeypatch):
"""Verify Omni initializes and tears down vLLM's distributed state properly.
This is a regression test for ensuring vLLM Omni also sets up vLLM's vars correctly
to avoid potential misalignment in cases where Omni uses vLLM's native coordinator,
e.g., MoE + diffusion.
"""
ensure_parallel_state_not_initialized()
monkeypatch.setattr(omni_parallel_state.current_omni_platform, "get_device_count", lambda: 1)
monkeypatch.setattr(omni_parallel_state.current_omni_platform, "set_device", lambda _device: None)
omni_parallel_state.init_distributed_environment(
world_size=1,
rank=0,
local_rank=0,
distributed_init_method=get_distributed_init_method(),
backend="gloo",
)
try:
ensure_parallel_state_initialized()
finally:
omni_parallel_state.destroy_distributed_env()
ensure_parallel_state_not_initialized()
def _fake_new_group_factory(created_groups: list[SimpleNamespace]):
def _fake_new_group(ranks, *args, **kwargs):
group = SimpleNamespace(ranks=list(ranks))
@@ -356,6 +356,94 @@ def test_attention_wrapper_rejects_spatial_extent_change(monkeypatch: pytest.Mon
wan_spatial_shard._SPATIAL_SHARD_CONTEXT.reset(token)
def _reference_conv_input(conv, x, cache_x):
"""The conv input the previous implementation built: cat, F.pad, then halo_exchange."""
padding = list(conv._padding)
if cache_x is not None and padding[4] > 0:
x = torch.cat([cache_x, x], dim=2)
padding[4] -= cache_x.shape[2]
x = torch.nn.functional.pad(x, padding)
x_padded, _, _ = wan_spatial_shard.halo_exchange(
x, group=object(), halo_size=conv.halo_size, split_dim=conv.split_dim
)
return x_padded, x.shape[conv.split_tensor_dim]
def _dist_conv(split_dim, kernel=(3, 3, 3), padding=(1, 1, 1)):
from diffusers.models.autoencoders.autoencoder_kl_wan import WanCausalConv3d
torch.manual_seed(0)
source = WanCausalConv3d(3, 4, kernel, padding=padding)
return source, wan_spatial_shard.WanDistCausalConv3d(source, object(), split_dim=split_dim)
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize("split_dim", ["width", "height"])
@pytest.mark.parametrize("with_cache", [False, True])
def test_single_pass_conv_input_matches_cat_pad_halo_on_one_rank(monkeypatch, split_dim, with_cache):
"""One rank, halo-bearing kernel: same bytes as the previous cat -> F.pad -> halo_exchange path."""
monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 1))
_, conv = _dist_conv(split_dim)
x = torch.randn(1, 3, 2, 6, 8)
cache_x = torch.randn(1, 3, 2, 6, 8) if with_cache else None
expected, expected_extent = _reference_conv_input(conv, x, cache_x)
got, got_extent = conv._assemble_input(x, cache_x)
assert got.shape == expected.shape and got_extent == expected_extent
assert torch.equal(got, expected)
@pytest.mark.core_model
@pytest.mark.cpu
def test_single_pass_conv_rejects_cache_longer_than_causal_padding():
_, conv = _dist_conv("width")
with pytest.raises(AssertionError, match="temporal cache exceeds"):
conv._assemble_input(torch.randn(1, 3, 2, 6, 8), torch.randn(1, 3, 3, 6, 8))
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize("split_dim", ["width", "height"])
def test_single_pass_conv_without_a_halo_matches_the_stock_causal_conv(monkeypatch, split_dim):
"""A kernel with no spatial extent along the split (the temporal conv) needs no halo and equals the stock conv."""
monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 1))
source, conv = _dist_conv(split_dim, kernel=(3, 1, 1), padding=(1, 0, 0))
x = torch.randn(1, 3, 2, 6, 8)
cache_x = torch.randn(1, 3, 2, 6, 8)
assert torch.equal(conv(x, cache_x), source(x, cache_x))
assert torch.equal(conv(x, None), source(x, None))
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize("split_dim", ["width", "height"])
@pytest.mark.parametrize("rank", [0, 1])
@pytest.mark.parametrize("kernel,padding", [((3, 3, 3), (1, 1, 1)), ((3, 1, 1), (0, 0, 0))])
def test_single_pass_conv_input_matches_cat_pad_halo_on_two_ranks(monkeypatch, split_dim, rank, kernel, padding):
"""Same bytes as cat -> F.pad -> halo_exchange, halos included, on either rank of a pair."""
monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (rank, 2))
def fake_p2p(*, rank, world_size, group, top_row_ref, bottom_row_ref, recv_top_buf, recv_bottom_buf):
# A neighbour's halo is whatever it sent; stand in with a marker derived from our own edge so
# both the reference and the single-pass path see the same exchange. Global edges stay zero.
recv_top_buf.copy_(top_row_ref * 0 + 7.0) if rank > 0 else recv_top_buf.zero_()
recv_bottom_buf.copy_(bottom_row_ref * 0 + 9.0) if rank < world_size - 1 else recv_bottom_buf.zero_()
monkeypatch.setattr(wan_spatial_shard, "_halo_exchange_p2p", fake_p2p)
_, conv = _dist_conv(split_dim, kernel=kernel, padding=padding)
x = torch.randn(1, 3, 1, 6, 8)
cache_x = torch.randn(1, 3, 2, 6, 8)
expected, expected_extent = _reference_conv_input(conv, x, cache_x)
got, got_extent = conv._assemble_input(x, cache_x)
assert got.shape == expected.shape and got_extent == expected_extent
assert torch.equal(got, expected)
@pytest.mark.core_model
@pytest.mark.cpu
def test_halo_exchange_single_rank_noop(monkeypatch: pytest.MonkeyPatch):
@@ -41,13 +41,13 @@ from vllm_omni.diffusion.distributed.parallel_state import (
initialize_model_parallel,
model_parallel_is_initialized,
)
from vllm_omni.diffusion.layers.mot.mot_qkv_parallel_linear import (
from vllm_omni.diffusion.models.bagel.mot.mot_qkv_parallel_linear import (
MoTQKVParallelLinear,
)
from vllm_omni.diffusion.layers.mot.mot_row_parallel_linear import (
from vllm_omni.diffusion.models.bagel.mot.mot_row_parallel_linear import (
MoTRowParallelLinear,
)
from vllm_omni.diffusion.layers.mot.ops.mot_gemm import (
from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import (
get_best_mot_config,
)
+4 -1
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# ruff: noqa: N803, E741
"""Layer-level correctness & performance test for MoTRMSNorm.
@@ -20,7 +23,7 @@ import torch
from vllm import ir
from vllm.config import VllmConfig, set_current_vllm_config
from vllm_omni.diffusion.layers.mot.mot_layernorm import MoTRMSNorm
from vllm_omni.diffusion.models.bagel.mot.mot_layernorm import MoTRMSNorm
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.gpu]
@@ -1,8 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
import vllm_omni.diffusion.layers.custom_op as custom_op
from vllm_omni.diffusion.layers.mot.mot_layernorm import MoTRMSNorm
from vllm_omni.diffusion.models.bagel.mot.mot_layernorm import MoTRMSNorm
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
+2 -2
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for the CustomOp platform-dispatch base class."""
import pytest
@@ -39,7 +39,7 @@ def test_forward_cuda_still_requires_an_implementation():
def test_mot_rmsnorm_dispatches_on_xpu_without_an_override():
"""MoTRMSNorm relies on the inherited fallback (regression test)."""
from vllm_omni.diffusion.layers.mot.mot_layernorm import MoTRMSNorm
from vllm_omni.diffusion.models.bagel.mot.mot_layernorm import MoTRMSNorm
assert "forward_xpu" not in MoTRMSNorm.__dict__
@@ -173,3 +173,20 @@ def test_fused_qk_norm_rope_min_tokens_resolution(monkeypatch):
monkeypatch.setenv(env, bad)
with pytest.raises(ValueError, match=env):
fused_qk_norm_rope_min_tokens(2048)
@pytest.mark.skipif(not torch.cuda.is_available() or not HAS_TRITON, reason="CUDA and Triton required")
@pytest.mark.parametrize("interleaved", [False, True])
def test_fused_qk_norm_rope_large_storage_offsets(interleaved):
"""Only three tokens, but the last row lies beyond signed 32-bit indexing."""
from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
# 4 GiB of backing storage; initialize only the three small visible rows.
q = torch.empty_strided((3, 1, 128), (2**30, 128, 1), device="cuda", dtype=torch.bfloat16)
q.fill_(1)
weight = torch.ones(128, device="cuda", dtype=torch.bfloat16)
table = torch.zeros(3, 96, device="cuda", dtype=torch.bfloat16)
table[:, :48] = 1
actual = fused_qk_norm_rope(q, q, weight, weight, table, _EPS, interleaved=interleaved)
for value in actual:
torch.testing.assert_close(value, torch.ones_like(value))
@@ -0,0 +1,125 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from diffusers.models.embeddings import apply_rotary_emb
from vllm.triton_utils import HAS_TRITON
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
def test_fused_qk_rope_fake_and_cpu_eligibility():
from vllm_omni.diffusion.layers.fused_qk_rope import (
_fused_qk_rope_fake,
fused_qk_rope,
fused_qk_rope_supported,
)
q = torch.randn(2, 3, 4, 8, dtype=torch.bfloat16)
k = torch.randn_like(q)
cos = torch.randn(3, 8)
sin = torch.randn_like(cos)
fake_q, fake_k = _fused_qk_rope_fake(q, k, cos, sin)
assert fake_q.shape == q.shape and fake_q.dtype == q.dtype
assert fake_k.shape == k.shape and fake_k.dtype == k.dtype
assert not fused_qk_rope_supported(q, k, cos, sin)
with pytest.raises(ValueError, match="contiguous CUDA BF16"):
fused_qk_rope(q, k, cos, sin)
def _cuda_inputs(batch: int, sequence: int, heads: int = 24, head_dim: int = 128):
q = torch.randn(batch, sequence, heads, head_dim, device="cuda", dtype=torch.bfloat16)
k = torch.randn_like(q)
# Deliberately do not repeat adjacent table columns. Diffusers consumes
# full-width tables, so even and odd outputs may use different values.
cos = torch.randn(sequence, head_dim, device="cuda", dtype=torch.float32)
sin = torch.randn_like(cos)
return q, k, cos, sin
def _reference(q, k, cos, sin):
rotary_emb = (cos, sin)
return (
apply_rotary_emb(q, rotary_emb, sequence_dim=1),
apply_rotary_emb(k, rotary_emb, sequence_dim=1),
)
@pytest.mark.cuda
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
@pytest.mark.parametrize("sequence", [512, 4096, 4608])
def test_fused_qk_rope_is_bit_exact_at_longcat_production_shapes(sequence):
from vllm_omni.diffusion.layers.fused_qk_rope import _launch_fused_qk_rope, fused_qk_rope
torch.manual_seed(sequence)
with torch.inference_mode():
q, k, cos, sin = _cuda_inputs(1, sequence)
expected = _reference(q, k, cos, sin)
for actual in (
_launch_fused_qk_rope(q, k, cos, sin),
fused_qk_rope(q, k, cos, sin),
):
assert torch.equal(actual[0], expected[0])
assert torch.equal(actual[1], expected[1])
@pytest.mark.cuda
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
def test_fused_qk_rope_is_bit_exact_for_batch_and_full_width_odd_even_tables():
from vllm_omni.diffusion.layers.fused_qk_rope import fused_qk_rope
torch.manual_seed(11)
with torch.inference_mode():
q, k, cos, sin = _cuda_inputs(2, 17, heads=3)
assert not torch.equal(cos[:, ::2], cos[:, 1::2])
assert not torch.equal(sin[:, ::2], sin[:, 1::2])
expected = _reference(q, k, cos, sin)
actual = fused_qk_rope(q, k, cos, sin)
assert torch.equal(actual[0], expected[0])
assert torch.equal(actual[1], expected[1])
@pytest.mark.cuda
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
def test_fused_qk_rope_rejects_noncontiguous_and_non_fp32_tables():
from vllm_omni.diffusion.layers.fused_qk_rope import fused_qk_rope_supported
q, k, cos, sin = _cuda_inputs(1, 8, heads=2)
q_noncontiguous = torch.randn(1, 8, 2, 256, device="cuda", dtype=torch.bfloat16)[..., ::2]
k_noncontiguous = torch.randn_like(q_noncontiguous)
cos_noncontiguous = torch.randn(8, 256, device="cuda")[:, ::2]
sin_noncontiguous = torch.randn(8, 256, device="cuda")[:, ::2]
with torch.inference_mode():
assert fused_qk_rope_supported(q, k, cos, sin)
assert not fused_qk_rope_supported(q_noncontiguous, k_noncontiguous, cos, sin)
assert not fused_qk_rope_supported(q, k, cos_noncontiguous, sin_noncontiguous)
assert not fused_qk_rope_supported(q, k, cos.to(torch.bfloat16), sin.to(torch.bfloat16))
q_wide = torch.randn(1, 8, 2, 256, device="cuda", dtype=torch.bfloat16)
k_wide = torch.randn_like(q_wide)
cos_wide = torch.randn(8, 256, device="cuda")
sin_wide = torch.randn_like(cos_wide)
assert not fused_qk_rope_supported(q_wide, k_wide, cos_wide, sin_wide)
@pytest.mark.cuda
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
def test_fused_qk_rope_custom_op_has_fullgraph_fake():
from vllm_omni.diffusion.layers.fused_qk_rope import fused_qk_rope
torch.manual_seed(29)
with torch.inference_mode():
q, k, cos, sin = _cuda_inputs(1, 19, heads=3)
expected = _reference(q, k, cos, sin)
compiled = torch.compile(fused_qk_rope, dynamic=True, fullgraph=True)
actual = compiled(q, k, cos, sin)
assert torch.equal(actual[0], expected[0])
assert torch.equal(actual[1], expected[1])
+79
View File
@@ -18,6 +18,7 @@ from vllm_omni.diffusion.lora.loader import (
WanLoraLoaderMixin,
_prepare_lora_delta,
_remap_state_dict_keys,
_resolve_lora_compute_device,
)
HEAD_DIM = 24
@@ -521,3 +522,81 @@ def test_prepare_lora_delta_with_compute_device():
assert delta is not None
assert delta.shape == (HEAD_DIM, HEAD_DIM)
assert len(used_keys) == 2
def test_resolve_lora_compute_device_probes_unusable_accelerator(mocker: MockerFixture):
"""cuda:N/npu:N may be reported without a physical device; the probe must reject it."""
mocker.patch(
"vllm_omni.diffusion.lora.loader.get_local_device",
return_value=torch.device("cuda:999"),
)
assert _resolve_lora_compute_device() is None
def test_load_lora_into_module_cpu_weights_match_cpu_reference(mocker: MockerFixture):
"""Regression test: params on CPU used to force compute_device=None and skip the
#7005 accelerator path. With an unusable accelerator, fusion must fall back to
CPU and still produce base + lora_B @ lora_A."""
pipeline = DummyPipeline(NUM_LAYERS, HEAD_DIM)
original_params = {name: param.clone() for name, param in pipeline.transformer.named_parameters()}
lora_state_dict = make_lora_state_dict_for_module(pipeline.transformer)
lora_state_dict = _remap_state_dict_keys(lora_state_dict, [(".to_out.0.", ".to_out.")])
mocker.patch(
"vllm_omni.diffusion.lora.loader.get_local_device",
return_value=torch.device("cuda:999"),
)
used_keys = pipeline.load_lora_into_module(lora_state_dict, pipeline.transformer)
assert len(used_keys) > 0
assert not (set(lora_state_dict.keys()) - used_keys)
for name, param in pipeline.transformer.named_parameters():
if not name.endswith(".weight"):
continue
base_key = name[: -len(".weight")]
key_a = f"{base_key}.lora_A.weight"
key_b = f"{base_key}.lora_B.weight"
if key_a in lora_state_dict and key_b in lora_state_dict:
expected = original_params[name] + (
lora_state_dict[key_b].to(param.dtype) @ lora_state_dict[key_a].to(param.dtype)
)
assert_close(param, expected)
def test_load_lora_into_module_forwards_usable_accel_device(mocker: MockerFixture):
"""A usable accelerator must be forwarded as compute_device for CPU params.
A spy forces the actual math back onto CPU so the test runs without a device."""
fake_dev = torch.device("cuda:0")
mocker.patch(
"vllm_omni.diffusion.lora.loader._resolve_lora_compute_device",
return_value=fake_dev,
)
seen_devices = []
original = _prepare_lora_delta
def spy(state_dict, base_key, *args, **kwargs):
seen_devices.append(kwargs.get("compute_device"))
kwargs["compute_device"] = None
return original(state_dict, base_key, *args, **kwargs)
mocker.patch(
"vllm_omni.diffusion.lora.loader._prepare_lora_delta",
side_effect=spy,
)
pipeline = DummyPipeline(NUM_LAYERS, HEAD_DIM)
original_params = {name: param.clone() for name, param in pipeline.transformer.named_parameters()}
lora_state_dict = make_lora_state_dict_for_module(pipeline.transformer)
lora_state_dict = _remap_state_dict_keys(lora_state_dict, [(".to_out.0.", ".to_out.")])
used_keys = pipeline.load_lora_into_module(lora_state_dict, pipeline.transformer)
# Every CPU param with a LoRA delta must have received the accelerator device.
assert seen_devices
assert all(d == fake_dev for d in seen_devices)
assert not (set(lora_state_dict.keys()) - used_keys)
assert_params_not_equal(pipeline.transformer, original_params)
@@ -1241,6 +1241,169 @@ def test_dlo_allgather_rejects_unvalidated_online_quant_method(monkeypatch):
loader.load_model(load_device="cpu")
def test_dlo_allgather_allows_unquantized_host_fallback():
"""Host-loaded unquantized fallback layers are plain contiguous bf16 —
the same runtime layout DLO already shards on the ordinary path — so the
allowlist must not reject them."""
from vllm_omni.quantization.int8_config import UnquantizedHostLinearMethod
model = nn.Module()
model.transformer = nn.Linear(2, 2, bias=False)
model.transformer.quant_method = object.__new__(UnquantizedHostLinearMethod)
assert DiffusersPipelineLoader._unsupported_dlo_allgather_online_quant_methods(model) == ()
def test_dlo_load_model_keeps_host_fallback_on_cpu_through_post_load_sweep(monkeypatch, mocker):
"""Through load_model(): DLO + online quant must build the model inside
load_unquantizable_fallback_on_cpu(), and the over-wide fallback weight
must survive the post-load sweep (the _process_weights_after_loading pass
plus model.to("cpu")) as the same host tensor it was loaded into — never
bounced to the accelerator."""
import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod
from vllm_omni.quantization import int8_config
from vllm_omni.quantization.int8_config import (
NPU_QUANT_MATMUL_MAX_OUT_FEATURES,
DiffusionInt8Config,
NPUInt8OnlineLinearMethod,
UnquantizedHostLinearMethod,
)
# Same stand-in as TestHostFallbackLoading: the eager fallback delegates to
# UnquantizedLinearMethod, which reads the TP group.
mock_group = mocker.Mock()
mock_group.rank_in_group = 0
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=1)
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_rank", return_value=0)
mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group)
od_config = _make_dlo_online_quant_config()
loader = DiffusersPipelineLoader(LoadConfig(), od_config)
out_features = NPU_QUANT_MATMUL_MAX_OUT_FEATURES + 1
built_with_ctx: list[bool] = []
def copy_loader(param, loaded_weight, *args, **kwargs):
param.data.copy_(loaded_weight)
def build_model(*_args, **_kwargs):
built_with_ctx.append(int8_config._LOAD_UNQUANTIZABLE_FALLBACK_ON_CPU.get())
quant_config = DiffusionInt8Config(is_checkpoint_int8_serialized=False, activation_scheme="dynamic")
method = NPUInt8OnlineLinearMethod(quant_config)
layer = nn.Module()
layer.quant_method = method
method.create_weights(
layer,
input_size_per_partition=8,
output_partition_sizes=[out_features],
input_size=8,
output_size=out_features,
params_dtype=torch.bfloat16,
weight_loader=copy_loader,
)
model = nn.Module()
model.transformer = layer
return model
def load_weights(_model, **_kwargs):
layer = _model.transformer
loaded = torch.arange(out_features * 8, dtype=torch.float32).reshape(out_features, 8).to(torch.bfloat16)
layer.weight.weight_loader(layer.weight, loaded)
loader._init_from_load_format = build_model # type: ignore[method-assign]
loader.load_weights = load_weights # type: ignore[method-assign]
loader._apply_skip_softmax_calibration = lambda _model: None # type: ignore[method-assign]
monkeypatch.setattr(
loader_mod,
"build_checkpoint_mmap_plan",
lambda *_args, **_kwargs: HostWeightPlanResult(
None,
"online quantization requires the ordinary loader",
),
)
# Diffusion DiT models have no vLLM Attention layers, so the upstream
# layerwise finalize is a no-op here; stub it to keep the real
# _process_weights_after_loading sweep CPU-only.
monkeypatch.setattr(
"vllm.model_executor.model_loader.reload.layerwise.finalize_layerwise_processing",
lambda *_args, **_kwargs: None,
)
model = loader.load_model(load_device="cpu", device=torch.device("cpu"))
# Construction ran inside load_unquantizable_fallback_on_cpu(), so the
# over-wide layer took the host-loading fallback...
assert built_with_ctx == [True]
assert type(model.transformer.quant_method) is UnquantizedHostLinearMethod
# ...and the real post-load sweep skipped it via the fully-loaded flag and
# model.to("cpu") left the loaded host tensor untouched.
assert model.transformer._already_called_process_weights_after_loading
assert model.transformer.weight.device.type == "cpu"
expected = torch.arange(out_features * 8, dtype=torch.float32).reshape(out_features, 8).to(torch.bfloat16)
assert torch.equal(model.transformer.weight, expected)
@pytest.mark.parametrize(("offload_after_quant", "ctx_expected"), [(True, True), (False, False)])
def test_hsdp_enters_host_fallback_context_only_when_offloading_after_quant(mocker, offload_after_quant, ctx_expected):
"""The HSDP path must apply the same host-fallback bound as the ordinary
path: with a quant config it initializes on the accelerator, so the
load_unquantizable_fallback_on_cpu() context is what keeps over-wide
fallback weights off the device before sharding."""
import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod
from vllm_omni.diffusion.offloader.module_collector import PipelineModules
from vllm_omni.quantization import int8_config
od_config = SimpleNamespace(
dtype=torch.float32,
parallel_config=SimpleNamespace(
use_hsdp=True,
hsdp_replicate_size=1,
hsdp_shard_size=2,
),
quantization_config=None,
)
loader = DiffusersPipelineLoader(LoadConfig(), od_config)
loader.quant_config = object()
model = nn.Module()
model.transformer = nn.Linear(2, 2, bias=False)
ctx_seen: list[bool] = []
def build_model(*_args, **_kwargs):
ctx_seen.append(int8_config._LOAD_UNQUANTIZABLE_FALLBACK_ON_CPU.get())
return model
loader._init_from_load_format = build_model # type: ignore[method-assign]
loader.load_weights = lambda _model: None # type: ignore[method-assign]
loader._process_weights_after_loading = lambda *_args: None # type: ignore[method-assign]
mocker.patch.object(
loader_mod.ModuleDiscovery,
"discover",
return_value=PipelineModules(
dits=[model.transformer],
dit_names=["transformer"],
vaes=[],
encoders=[],
encoder_names=[],
resident_modules=[],
resident_names=[],
),
)
mocker.patch(
"vllm_omni.diffusion.quantization.hsdp_fp8.prepare_fp8_layers_for_fsdp",
side_effect=lambda _model: None,
)
mocker.patch.object(
loader_mod,
"apply_hsdp_to_model",
side_effect=lambda *_args, **_kwargs: None,
)
loader._load_model_with_hsdp(torch.device("cpu"), offload_after_quant=offload_after_quant)
assert ctx_seen == [ctx_expected]
def test_dlo_allgather_online_mxfp8_uses_ordinary_loader(monkeypatch):
mxfp8_config = pytest.importorskip("vllm_omni.quantization.mxfp8_config")
NPUMxfp8OnlineLinearMethod = mxfp8_config.NPUMxfp8OnlineLinearMethod
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+454
View File
@@ -0,0 +1,454 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import json
from pathlib import Path
from shutil import rmtree
from types import SimpleNamespace
import pytest
import torch
from safetensors.torch import save_file
from tests.model_tests.diffusion.anima_builder import CHECKPOINT_FILENAME, tiny_anima_builder
from vllm_omni.diffusion.attention import selector
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_qwenimage import DistributedAutoencoderKLQwenImage
from vllm_omni.diffusion.models.anima import pipeline_anima
from vllm_omni.diffusion.models.anima.anima_text_conditioner import AnimaTextConditioner
from vllm_omni.diffusion.models.anima.anima_transformer import AnimaTransformer3DModel
from vllm_omni.diffusion.models.anima.pipeline_anima import AnimaPipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@pytest.mark.parametrize(
"offload_args",
[
{"enable_cpu_offload": True},
{"enable_layerwise_offload": True},
{"enable_distributed_layerwise_offload": True},
{"diffusion_offload_config": {"mode": "module", "components": ["dit"]}},
{
"diffusion_offload_config": {
"mode": "layer",
"components": ["dit"],
"layer_options": {"dit": {"weight_transfer": "rank-local"}},
}
},
],
)
def test_native_anima_rejects_offload(offload_args):
config = OmniDiffusionConfig(model="anima.safetensors", **offload_args)
with pytest.raises(NotImplementedError, match="offload"):
AnimaPipeline(od_config=config, device=torch.device("cpu"))
def test_native_anima_initializes_without_offload():
config = OmniDiffusionConfig(model="anima.safetensors")
AnimaPipeline(od_config=config, device=torch.device("cpu"))
def test_anima_registration() -> None:
"""The native Anima pipeline is registered and importable."""
from vllm_omni.diffusion.registry import DiffusionModelRegistry
assert DiffusionModelRegistry._try_load_model_cls("AnimaPipeline") is not None
@pytest.mark.parametrize("model_class_name", ["AnimaPipeline", "AnimaModularPipeline"])
def test_enrich_config_native_anima_checkpoint(tmp_path: Path, model_class_name: str) -> None:
"""Native and reference Anima class names resolve to AnimaPipeline."""
dummy_checkpoint = tmp_path / "model.safetensors"
dummy_checkpoint.write_text("dummy")
config = OmniDiffusionConfig(
model=str(dummy_checkpoint),
model_class_name=model_class_name,
)
config.enrich_config()
assert config.diffusion_load_format == "default"
assert config.model_class_name == "AnimaPipeline"
assert config.diffusers_pipeline_cls is None
def test_native_anima_component_paths_use_custom_pipeline_args() -> None:
"""Anima resolves shared and component-specific paths from native args."""
pipeline = AnimaPipeline.__new__(AnimaPipeline)
pipeline.od_config = SimpleNamespace(
custom_pipeline_args={
"components_path": "/tmp/anima-components",
"vae_path": "/tmp/anima-vae",
"text_encoder_model": "/tmp/anima-text-encoder",
}
)
assert pipeline._component_path("vae", "/tmp/default") == "/tmp/anima-vae"
assert pipeline._component_path("text_encoder", "/tmp/default") == "/tmp/anima-text-encoder"
assert pipeline._component_path("tokenizer", "/tmp/default") == "/tmp/default"
def test_native_anima_converts_original_cosmos_transformer_keys() -> None:
"""Original Cosmos checkpoint names map to native module names."""
converted = AnimaPipeline._convert_original_transformer_state_dict(
{
"net.x_embedder.proj.1.weight": "patch",
"net.blocks.0.self_attn.q_proj.weight": "q",
"net.blocks.0.self_attn.q_norm.weight": "q_norm",
"net.blocks.0.mlp.layer1.weight": "mlp",
"net.final_layer.linear.weight": "out",
"net.accum_iteration": "drop",
}
)
assert converted == {
"patch_embed.proj.weight": "patch",
"transformer_blocks.0.attn1.to_q.weight": "q",
"transformer_blocks.0.attn1.norm_q.weight": "q_norm",
"transformer_blocks.0.ff.net.0.proj.weight": "mlp",
"proj_out.weight": "out",
}
def test_native_anima_resolves_vae_scale_factor_from_loaded_vae() -> None:
"""The loaded VAE determines the spatial scale factor."""
from vllm_omni.diffusion.models.anima.pipeline_anima import _anima_vae_scale_factor_from_vae
vae = SimpleNamespace(config=SimpleNamespace(spatial_compression_ratio=16))
assert _anima_vae_scale_factor_from_vae(vae) == 16
@pytest.mark.parametrize("use_config_file", [False, True])
def test_native_anima_loads_synthetic_checkpoint(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, use_config_file: bool
) -> None:
"""Native modules load exact weights from an Anima safetensors checkpoint."""
tiny_transformer_config = {
"in_channels": 1,
"out_channels": 1,
"num_attention_heads": 1,
"attention_head_dim": 12,
"num_layers": 1,
"mlp_ratio": 1.0,
"text_embed_dim": 4,
"adaln_lora_dim": 3,
"max_size": (1, 2, 2),
"patch_size": (1, 1, 1),
"rope_scale": (1.0, 1.0, 1.0),
"concat_padding_mask": True,
"extra_pos_embed_type": None,
}
tiny_text_conditioner_config = {
"source_dim": 4,
"target_dim": 4,
"model_dim": 4,
"num_layers": 1,
"num_attention_heads": 1,
"target_vocab_size": 8,
"min_sequence_length": 4,
}
if use_config_file:
(tmp_path / "anima.json").write_text(
json.dumps({"transformer": tiny_transformer_config, "text_conditioner": tiny_text_conditioner_config})
)
else:
monkeypatch.setattr(pipeline_anima, "ANIMA_TRANSFORMER_CONFIG", tiny_transformer_config)
monkeypatch.setattr(pipeline_anima, "ANIMA_TEXT_CONDITIONER_CONFIG", tiny_text_conditioner_config)
monkeypatch.setattr(selector, "_cached_get_backend_cls", lambda *_args, **_kwargs: SDPABackend)
transformer = AnimaTransformer3DModel(**tiny_transformer_config)
text_conditioner = AnimaTextConditioner(**tiny_text_conditioner_config)
transformer_state = {name: tensor.detach().clone() for name, tensor in transformer.state_dict().items()}
text_conditioner_state = {name: tensor.detach().clone() for name, tensor in text_conditioner.state_dict().items()}
checkpoint_state = {
**{f"transformer.{name}": tensor for name, tensor in transformer_state.items()},
**{f"text_conditioner.{name}": tensor for name, tensor in text_conditioner_state.items()},
}
checkpoint_path = tmp_path / "anima.safetensors"
save_file(checkpoint_state, str(checkpoint_path))
pipeline = AnimaPipeline.__new__(AnimaPipeline)
pipeline.od_config = SimpleNamespace(model=str(checkpoint_path), dtype=torch.float32)
pipeline.device = torch.device("cpu")
def assert_loaded(loaded_transformer, loaded_text_conditioner):
for name, tensor in transformer_state.items():
assert torch.equal(loaded_transformer.state_dict()[name], tensor)
for name, tensor in text_conditioner_state.items():
assert torch.equal(loaded_text_conditioner.state_dict()[name], tensor)
loaded_transformer, loaded_text_conditioner = pipeline._load_native_denoiser_components(dict(checkpoint_state))
assert_loaded(loaded_transformer, loaded_text_conditioner)
loaded_transformer, loaded_text_conditioner = pipeline._load_native_denoiser_components()
assert_loaded(loaded_transformer, loaded_text_conditioner)
if use_config_file:
tiny_transformer_config["num_layers"] = 2
(tmp_path / "anima.json").write_text(
json.dumps({"transformer": tiny_transformer_config, "text_conditioner": tiny_text_conditioner_config})
)
with pytest.raises(RuntimeError, match="Missing key"):
pipeline._load_native_denoiser_components()
def test_tiny_anima_generates_images(monkeypatch: pytest.MonkeyPatch) -> None:
"""Load every saved component and exercise CFG, decoding, and multiple outputs on CPU."""
monkeypatch.setattr(selector, "_cached_get_backend_cls", lambda *_args, **_kwargs: SDPABackend)
monkeypatch.setattr(current_omni_platform, "is_available", lambda: False)
monkeypatch.setattr(
DistributedAutoencoderKLQwenImage,
"init_distributed",
lambda self: setattr(self, "distributed_executor", SimpleNamespace(parallel_size=1)),
)
model_dir = tiny_anima_builder()
try:
config = OmniDiffusionConfig(
model=str(Path(model_dir) / CHECKPOINT_FILENAME), model_class_name="AnimaPipeline", dtype=torch.float32
)
pipeline = AnimaPipeline(od_config=config, device=torch.device("cpu"))
pipeline.load_weights()
pipeline.eval()
outputs = []
with torch.inference_mode():
for _ in range(2):
request = OmniDiffusionRequest(
prompt="Dummy prompt",
request_id="tiny-anima",
sampling_params=OmniDiffusionSamplingParams(
height=32,
width=32,
num_inference_steps=2,
guidance_scale=4.0,
num_outputs_per_prompt=2,
generator=torch.Generator(device="cpu").manual_seed(42),
),
)
outputs.append(pipeline(DiffusionRequestBatch([request]))[0].output)
assert outputs[0].shape == (2, 3, 32, 32)
assert torch.isfinite(outputs[0]).all()
assert torch.equal(outputs[0], outputs[1])
assert not torch.equal(outputs[0][0], outputs[0][1])
finally:
rmtree(model_dir)
def test_native_anima_profiler_setup_is_deferred_and_idempotent(monkeypatch: pytest.MonkeyPatch) -> None:
"""Profiler targets are wrapped once, after deferred components are loaded."""
pipeline = AnimaPipeline.__new__(AnimaPipeline)
pipeline.od_config = SimpleNamespace(enable_diffusion_pipeline_profiler=True)
pipeline._profiler_initialized = False
pipeline.vae = SimpleNamespace(decode=lambda: "decoded")
pipeline.text_encoder = SimpleNamespace(forward=lambda: "encoded")
monkeypatch.setattr(current_omni_platform, "is_available", lambda: False)
pipeline._setup_profiler()
wrapped_decode = pipeline.vae.decode
wrapped_text_encoder = pipeline.text_encoder.forward
pipeline._setup_profiler()
assert pipeline.vae.decode is wrapped_decode
assert pipeline.text_encoder.forward is wrapped_text_encoder
assert pipeline.vae.decode() == "decoded"
assert pipeline.text_encoder.forward() == "encoded"
assert "AnimaPipeline.vae.decode" in pipeline.stage_durations
assert "AnimaPipeline.text_encoder.forward" in pipeline.stage_durations
def _make_anima_forward_probe():
pipeline = AnimaPipeline.__new__(AnimaPipeline)
pipeline.device = torch.device("cpu")
pipeline.vae_scale_factor = 8
pipeline.transformer = SimpleNamespace(dtype=torch.float32)
pipeline.text_encoder = SimpleNamespace(dtype=torch.float32)
pipeline._current_timestep = None
pipeline._num_timesteps = 0
pipeline._guidance_scale = 0.0
captured = {}
def encode_prompt(**_kwargs):
return {
"qwen_prompt_embeds": torch.zeros(1, 2, 4),
"qwen_attention_mask": torch.ones(1, 2),
"t5_input_ids": torch.ones(1, 2, dtype=torch.long),
"t5_attention_mask": torch.ones(1, 2),
"negative_qwen_prompt_embeds": torch.zeros(1, 2, 4),
"negative_qwen_attention_mask": torch.ones(1, 2),
"negative_t5_input_ids": torch.ones(1, 2, dtype=torch.long),
"negative_t5_attention_mask": torch.ones(1, 2),
}
def condition_prompt_embeds(**_kwargs):
return torch.zeros(1, 2, 4)
def prepare_latents(**kwargs):
captured["prepare_latents"] = kwargs
return torch.zeros(1, 16, 1, kwargs["height"] // 8, kwargs["width"] // 8)
def diffuse(**kwargs):
captured["diffuse"] = kwargs
return kwargs["latents"]
pipeline.encode_prompt = encode_prompt
pipeline.condition_prompt_embeds = condition_prompt_embeds
pipeline.prepare_latents = prepare_latents
pipeline.prepare_timesteps = lambda **_kwargs: (torch.ones(1), 1)
pipeline.diffuse = diffuse
def decode_latents(latents, output_type="pil"):
captured["output_type"] = output_type
return DiffusionOutput(output=latents)
pipeline.decode_latents = decode_latents
return pipeline, captured
def test_native_anima_forward_uses_official_default_resolution() -> None:
"""Forward uses Anima's reference resolution when none is requested."""
pipeline, captured = _make_anima_forward_probe()
req = OmniDiffusionRequest(
prompt="a red cube",
sampling_params=OmniDiffusionSamplingParams(),
request_id="anima-defaults",
)
pipeline.forward(DiffusionRequestBatch([req]))
assert captured["prepare_latents"]["height"] == 1024
assert captured["prepare_latents"]["width"] == 1024
def test_native_anima_forward_honors_sampling_output_type() -> None:
"""Request-level output type controls Anima decoding."""
pipeline, captured = _make_anima_forward_probe()
req = OmniDiffusionRequest(
prompt="a red cube",
sampling_params=OmniDiffusionSamplingParams(output_type="latent"),
request_id="anima-output-type",
)
outputs = pipeline.forward(DiffusionRequestBatch([req]))
assert len(outputs) == 1
assert isinstance(outputs[0], DiffusionOutput)
assert outputs[0].output.shape == (1, 16, 1, 128, 128)
assert captured["output_type"] == "latent"
@pytest.mark.parametrize("request_count", [0, 2])
def test_native_anima_forward_rejects_non_single_request_batches(request_count: int) -> None:
pipeline, captured = _make_anima_forward_probe()
requests = [
OmniDiffusionRequest(
prompt="a red cube",
sampling_params=OmniDiffusionSamplingParams(),
request_id=f"anima-{index}",
)
for index in range(request_count)
]
with pytest.raises(ValueError, match="supports one request per forward"):
pipeline.forward(DiffusionRequestBatch(requests))
assert captured == {}
def test_native_anima_explicit_guidance_scale_drives_cfg_multiplier() -> None:
"""An explicit guidance scale controls native classifier-free guidance."""
pipeline, captured = _make_anima_forward_probe()
req = OmniDiffusionRequest(
prompt="a red cube",
sampling_params=OmniDiffusionSamplingParams(guidance_scale=5.0),
request_id="anima-guidance",
)
pipeline.forward(DiffusionRequestBatch([req]))
assert captured["diffuse"]["do_true_cfg"] is True
assert captured["diffuse"]["true_cfg_scale"] == 5.0
def test_native_anima_true_cfg_scale_overrides_guidance_multiplier() -> None:
"""An explicit true-CFG scale overrides the guidance-scale multiplier."""
pipeline, captured = _make_anima_forward_probe()
req = OmniDiffusionRequest(
prompt="a red cube",
sampling_params=OmniDiffusionSamplingParams(guidance_scale=5.0, true_cfg_scale=3.0),
request_id="anima-true-cfg",
)
pipeline.forward(DiffusionRequestBatch([req]))
assert captured["diffuse"]["do_true_cfg"] is True
assert captured["diffuse"]["true_cfg_scale"] == 3.0
def test_native_anima_cfg_equation() -> None:
"""The denoising loop applies the standard true-CFG equation."""
pipeline = AnimaPipeline.__new__(AnimaPipeline)
pipeline.device = torch.device("cpu")
pipeline._interrupt = False
class MockProgressBar:
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def update(self):
pass
pipeline.progress_bar = lambda **_kwargs: MockProgressBar()
class MockTransformer:
dtype = torch.float32
def __call__(self, hidden_states, timestep, encoder_hidden_states, padding_mask, return_dict=False):
# Return double the encoder_hidden_states to simulate transformer output
return (encoder_hidden_states * 2.0,)
pipeline.transformer = MockTransformer()
class MockScheduler:
def set_begin_index(self, index):
pass
def step(self, noise_pred, t, latents, return_dict=False):
# Return the noise_pred itself to inspect it
return (noise_pred,)
pipeline.scheduler = MockScheduler()
prompt_embeds = torch.tensor([[[2.0, 3.0]]])
negative_prompt_embeds = torch.tensor([[[1.0, 2.0]]])
latents = torch.zeros(1, 16, 1, 16, 16)
padding_mask = torch.zeros(1, 1, 16, 16)
timesteps = torch.tensor([500.0])
# Cond = prompt_embeds * 2.0 = [4.0, 6.0]
# Uncond = negative_prompt_embeds * 2.0 = [2.0, 4.0]
# Expected noise_pred = uncond + true_cfg_scale * (cond - uncond)
# = [2.0, 4.0] + 4.0 * ([4.0, 6.0] - [2.0, 4.0])
# = [2.0, 4.0] + 4.0 * [2.0, 2.0]
# = [2.0, 4.0] + [8.0, 8.0] = [10.0, 12.0]
out_latents = pipeline.diffuse(
prompt_embeds=prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds,
latents=latents,
padding_mask=padding_mask,
timesteps=timesteps,
do_true_cfg=True,
true_cfg_scale=4.0,
)
expected = torch.tensor([[[10.0, 12.0]]])
assert torch.allclose(out_latents, expected)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Focused CPU tests for BAGEL step execution and packed batching."""
from __future__ import annotations
@@ -18,12 +18,14 @@ from vllm_omni.diffusion.models.bagel.bagel_transformer import Bagel, NaiveCache
from vllm_omni.diffusion.models.bagel.pipeline_bagel import (
BagelGenParams,
BagelPipeline,
_bagel_canvas_requested,
get_bagel_pre_process_func,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.sched import StepScheduler
from vllm_omni.diffusion.worker.input_batch import InputBatch
from vllm_omni.diffusion.worker.utils import StepRequestState
from vllm_omni.errors import OmniClientError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -118,7 +120,7 @@ def test_bagel_step_preprocessor_separates_incompatible_cfg_settings(first_extra
assert first.batch_compatibility_key != second.batch_compatibility_key
def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
def _pre_process_with_stub_checkpoint(tmp_path):
(tmp_path / "config.json").write_text(
'{"vae_config":{"downsample":8},"latent_patch_size":2,"max_latent_size":32}',
encoding="utf-8",
@@ -131,9 +133,22 @@ def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
'{"weight_map":{"latent_pos_embed.pos_embed":"ema.safetensors"}}',
encoding="utf-8",
)
pre_process = get_bagel_pre_process_func(
types.SimpleNamespace(model=str(tmp_path), revision=None, step_execution=True)
return get_bagel_pre_process_func(types.SimpleNamespace(model=str(tmp_path), revision=None, step_execution=True))
def _edit_request(source_size, sampling_params, request_id="req"):
return OmniDiffusionRequest(
prompt={
"prompt": "edit this image",
"multi_modal_data": {"image": [Image.new("RGB", source_size)]},
},
sampling_params=sampling_params,
request_id=request_id,
)
def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
requests = [
OmniDiffusionRequest(
prompt={
@@ -163,6 +178,172 @@ def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
assert schedule.num_waiting_reqs == 1
def test_bagel_pre_process_keeps_explicitly_requested_canvas(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = _edit_request((512, 1024), OmniDiffusionSamplingParams(num_inference_steps=2, height=1024, width=1024))
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
assert _bagel_canvas_requested(request.sampling_params)
def test_bagel_pre_process_aligns_source_derived_canvas_for_size_auto(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = _edit_request(
(500, 700),
OmniDiffusionSamplingParams(
num_inference_steps=2,
height=700,
width=500,
height_not_provided=True,
width_not_provided=True,
),
)
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (704, 496)
assert not _bagel_canvas_requested(request.sampling_params)
def test_bagel_pre_process_derives_canvas_when_caller_asked_for_none(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = _edit_request((800, 400), OmniDiffusionSamplingParams(num_inference_steps=2))
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (400, 800)
assert (request.sampling_params.height_not_provided, request.sampling_params.width_not_provided) == (True, True)
assert not _bagel_canvas_requested(request.sampling_params)
def test_bagel_pre_process_marks_kv_metadata_canvas_as_requested(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = _edit_request(
(500, 700),
OmniDiffusionSamplingParams(
num_inference_steps=2,
height=700,
width=500,
height_not_provided=True,
width_not_provided=True,
kv_metadata={"image_shape": (1024, 1024)},
),
)
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
assert (request.sampling_params.height_not_provided, request.sampling_params.width_not_provided) == (False, False)
assert _bagel_canvas_requested(request.sampling_params)
def test_bagel_pre_process_prefers_explicit_canvas_over_kv_metadata(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = _edit_request(
(1280, 720),
OmniDiffusionSamplingParams(
num_inference_steps=2,
height=1024,
width=1024,
kv_metadata={"image_shape": (576, 1024)},
),
)
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
assert _bagel_canvas_requested(request.sampling_params)
def test_bagel_pre_process_floors_unaligned_explicit_canvas_with_warning(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = _edit_request(
(1280, 720),
OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
)
with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.logger.warning") as warning:
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (688, 992)
assert _bagel_canvas_requested(request.sampling_params)
warning.assert_called_once()
assert "width 1000->992, height 700->688" in warning.call_args.args[-1]
def test_bagel_pre_process_keeps_aligned_explicit_canvas_silent(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = _edit_request(
(1280, 720),
OmniDiffusionSamplingParams(num_inference_steps=2, height=512, width=1024),
)
with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.logger.warning") as warning:
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (512, 1024)
warning.assert_not_called()
def test_bagel_pre_process_rejects_explicit_canvas_over_checkpoint_limit(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = _edit_request(
(1280, 720),
OmniDiffusionSamplingParams(num_inference_steps=2, height=512, width=2048),
)
with pytest.raises(OmniClientError, match="width=2048 exceeds the BAGEL checkpoint limit of 1024") as excinfo:
pre_process(request)
assert excinfo.value.status_code == 400
def test_bagel_pre_process_aligns_text2img_canvas_too(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = OmniDiffusionRequest(
prompt="a cat",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
request_id="req",
)
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (688, 992)
def test_bagel_pre_process_skips_canvas_alignment_for_text_output(tmp_path):
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
request = OmniDiffusionRequest(
prompt={
"prompt": "describe this image",
"modalities": ["text"],
"multi_modal_data": {"image": [Image.new("RGB", (1280, 720))]},
},
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
request_id="req",
)
pre_process(request)
assert (request.sampling_params.height, request.sampling_params.width) == (700, 1000)
assert request.use_step_execution is False
@pytest.mark.parametrize(
("sampling_kwargs", "requested"),
[
({"height": 1024, "width": 1024}, True),
({"height": 1024, "width": 512, "height_not_provided": True, "width_not_provided": True}, False),
({}, False),
],
ids=["explicit_size", "size_auto", "unset"],
)
def test_bagel_canvas_requested_separates_asked_for_from_derived(sampling_kwargs, requested):
assert _bagel_canvas_requested(OmniDiffusionSamplingParams(**sampling_kwargs)) is requested
def _cache(length: int, value: float, num_layers: int = 1) -> NaiveCache:
cache = NaiveCache(num_layers)
if length:
@@ -13,11 +13,13 @@ Two groups:
CFG handling, and reshape logic can be verified numerically on CPU.
"""
import os
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from transformers import Qwen3VLConfig
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig, TransformerConfig
@@ -56,9 +58,17 @@ def mock_dependencies(mocker, monkeypatch):
f"{_MODULE}.FlowMatchEulerDiscreteScheduler.from_pretrained",
lambda *a, **k: mock_scheduler,
)
monkeypatch.setattr(
mllm_loader = mocker.patch(
f"{_MODULE}.Qwen3VLForConditionalGeneration.from_pretrained",
lambda *a, **k: mllm_wrapper,
return_value=mllm_wrapper,
)
mllm_config_loader = mocker.patch(
f"{_MODULE}.Qwen3VLConfig.from_pretrained",
return_value=Qwen3VLConfig(),
)
mllm_builder = mocker.patch(
f"{_MODULE}.create_transformers_model_with_vllm_linears",
return_value=nn.Linear(1, 1),
)
monkeypatch.setattr(
f"{_MODULE}.Qwen3VLProcessor.from_pretrained",
@@ -74,12 +84,17 @@ def mock_dependencies(mocker, monkeypatch):
mock_transformer_cls.return_value = mock_transformer_instance
monkeypatch.setattr(f"{_MODULE}.BooguImageTransformer2DModel", mock_transformer_cls)
# Treat the dummy model id as a local path: skips hub prefetch.
mocker.patch("os.path.exists", return_value=True)
# Treat only the dummy model id as local. Other filesystem checks (for
# example lazy imports in the quantization registry) must remain real.
path_exists = os.path.exists
mocker.patch("os.path.exists", side_effect=lambda path: str(path).startswith("dummy-boogu") or path_exists(path))
return {
"inner_encoder": inner_encoder,
"mllm_wrapper": mllm_wrapper,
"mllm_loader": mllm_loader,
"mllm_config_loader": mllm_config_loader,
"mllm_builder": mllm_builder,
"processor": mock_processor,
"vae": mock_vae,
"scheduler": mock_scheduler,
@@ -128,6 +143,9 @@ def test_constructor_wires_components(boogu_pipeline, mock_dependencies):
assert boogu_pipeline.vae_scale_factor == 8
assert boogu_pipeline.default_sample_size == 128
assert hasattr(boogu_pipeline, "load_weights")
assert "quantization_config" not in mock_dependencies["mllm_loader"].call_args.kwargs
mock_dependencies["mllm_config_loader"].assert_not_called()
mock_dependencies["mllm_builder"].assert_not_called()
def test_constructor_strips_mllm_lm_head(boogu_pipeline, mock_dependencies):
@@ -188,7 +206,7 @@ def test_constructor_weights_sources(boogu_pipeline):
assert source.fall_back_to_pt is True
def test_constructor_routes_only_transformer_config(mock_dependencies, mocker):
def test_constructor_rejects_unsupported_mllm_quantization(mock_dependencies, mocker):
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
@@ -204,10 +222,143 @@ def test_constructor_routes_only_transformer_config(mock_dependencies, mocker):
{"transformer": transformer_config, "mllm": encoder_config, "vae": None}
),
)
with pytest.raises(ValueError, match="Boogu MLLM only supports FP8 quantization"):
BooguImagePipeline(od_config=od_config)
mock_dependencies["mllm_loader"].assert_not_called()
mock_dependencies["mllm_config_loader"].assert_not_called()
@pytest.mark.parametrize(
("quantization_config", "quantize_mllm"),
[
pytest.param("fp8", True, id="global-fp8"),
pytest.param({"mllm": {"method": "fp8"}, "transformer": {"method": "fp8"}}, True, id="mllm-and-dit-fp8"),
pytest.param({"mllm": None, "transformer": "fp8"}, False, id="dit-only-fp8"),
],
)
def test_constructor_routes_mllm_quantization(mock_dependencies, quantization_config, quantize_mllm):
from transformers import AutoModel
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import kFp8Static128BlockSym
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
od_config = OmniDiffusionConfig(
model="dummy-boogu",
revision="test-revision",
tf_model_config=TransformerConfig(params={}),
dtype=torch.bfloat16,
quantization_config=quantization_config,
)
pipeline = BooguImagePipeline(od_config=od_config)
if quantize_mllm:
mock_dependencies["mllm_loader"].assert_not_called()
mock_dependencies["mllm_config_loader"].assert_called_once_with(
"dummy-boogu", subfolder="mllm", local_files_only=True, revision="test-revision"
)
builder = mock_dependencies["mllm_builder"]
builder.assert_called_once()
auto_cls, hf_config, online_quant_config = builder.call_args.args
assert auto_cls is AutoModel
assert hf_config is mock_dependencies["mllm_config_loader"].return_value
assert isinstance(online_quant_config, OnlineQuantizationConfig)
assert online_quant_config.args.linear.weight == kFp8Static128BlockSym
assert builder.call_args.kwargs == {
"dtype": torch.bfloat16,
"device": pipeline._execution_device,
"prefix": "mllm",
"skip_modules": ("mllm.visual",),
}
assert pipeline.mllm is builder.return_value
assert not pipeline.mllm.training
assert all(not parameter.requires_grad for parameter in pipeline.mllm.parameters())
transformer_source, mllm_source = pipeline.weights_sources
assert transformer_source.subfolder == "transformer"
assert mllm_source.model_or_path == "dummy-boogu"
assert mllm_source.subfolder == "mllm"
assert mllm_source.prefix == "mllm."
assert mllm_source.revision == "test-revision"
else:
assert "quantization_config" not in mock_dependencies["mllm_loader"].call_args.kwargs
assert pipeline.mllm is mock_dependencies["inner_encoder"]
mock_dependencies["mllm_config_loader"].assert_not_called()
mock_dependencies["mllm_builder"].assert_not_called()
assert [source.subfolder for source in pipeline.weights_sources] == ["transformer"]
assert isinstance(mock_dependencies["transformer_cls"].call_args.kwargs["quant_config"], Fp8Config)
def test_constructor_preserves_mllm_ignored_layers(mock_dependencies):
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
quant_config = Fp8Config(
ignored_layers=["mllm.language_model.layers.0.self_attn.q_proj", "transformer.blocks.0.attn.to_q"]
)
od_config = OmniDiffusionConfig(
model="dummy-boogu",
tf_model_config=TransformerConfig(params={}),
dtype=torch.bfloat16,
quantization_config=quant_config,
)
BooguImagePipeline(od_config=od_config)
kwargs = mock_dependencies["transformer_cls"].call_args.kwargs
assert kwargs["quant_config"] is transformer_config
assert kwargs["prefix"] == "transformer"
online_quant_config = mock_dependencies["mllm_builder"].call_args.args[2]
assert online_quant_config.ignored_layers == quant_config.ignored_layers
def test_constructor_preserves_serialized_mllm_quantization(mock_dependencies):
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
mock_dependencies["mllm_config_loader"].return_value = Qwen3VLConfig(
quantization_config={"quant_method": "fp8", "modules_to_not_convert": ["model.visual"]}
)
od_config = OmniDiffusionConfig(
model="dummy-boogu-fp8",
tf_model_config=TransformerConfig(params={}),
dtype=torch.bfloat16,
quantization_config="fp8",
)
pipeline = BooguImagePipeline(od_config=od_config)
# No override: HF reads the serialized checkpoint's scales and skip list.
assert "quantization_config" not in mock_dependencies["mllm_loader"].call_args.kwargs
assert pipeline.mllm is mock_dependencies["inner_encoder"]
mock_dependencies["mllm_builder"].assert_not_called()
assert [source.subfolder for source in pipeline.weights_sources] == ["transformer"]
def test_load_weights_maps_mllm_and_preserves_transformer():
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
pipeline = object.__new__(BooguImagePipeline)
nn.Module.__init__(pipeline)
pipeline.mllm = nn.ModuleDict(
{
"language_model": nn.Linear(2, 2, bias=False),
"visual": nn.Linear(2, 2, bias=False),
}
)
pipeline.transformer = nn.Linear(2, 2, bias=False)
language_weight = torch.full((2, 2), 1.0)
visual_weight = torch.full((2, 2), 2.0)
transformer_weight = torch.full((2, 2), 3.0)
loaded = pipeline.load_weights(
[
("mllm.lm_head.weight", torch.full((2, 2), 9.0)),
("mllm.model.language_model.weight", language_weight),
("mllm.model.visual.weight", visual_weight),
("transformer.weight", transformer_weight),
]
)
assert loaded == {"mllm.language_model.weight", "mllm.visual.weight", "transformer.weight"}
torch.testing.assert_close(pipeline.mllm["language_model"].weight, language_weight)
torch.testing.assert_close(pipeline.mllm["visual"].weight, visual_weight)
torch.testing.assert_close(pipeline.transformer.weight, transformer_weight)
@pytest.mark.parametrize(
@@ -0,0 +1,177 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from vllm.triton_utils import HAS_TRITON
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.models.ernie_image.ernie_image_transformer import _apply_rotary_emb
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
@pytest.fixture(autouse=True)
def reset_fused_rope_state():
from vllm_omni.diffusion.models.ernie_image import fused_rope
fused_rope._FAILED_KEYS.clear()
yield
fused_rope._FAILED_KEYS.clear()
def _inputs(shape: tuple[int, int, int, int]):
batch, sequence, _, head_dim = shape
torch.manual_seed(17)
query = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
key = torch.randn_like(query)
freqs_cos = torch.randn(
batch,
sequence,
head_dim // 2,
device="cuda",
dtype=torch.float32,
)
freqs_sin = torch.randn_like(freqs_cos)
return query, key, freqs_cos, freqs_sin
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
@pytest.mark.parametrize(
"shape",
[
(1, 1, 1, 128),
(2, 257, 7, 128),
(2, 4608, 32, 128),
],
)
def test_fused_qk_rope_is_bit_exact(shape):
from vllm_omni.diffusion.models.ernie_image import fused_rope
query, key, freqs_cos, freqs_sin = _inputs(shape)
with torch.inference_mode():
expected_query = _apply_rotary_emb(query, freqs_cos, freqs_sin)
expected_key = _apply_rotary_emb(key, freqs_cos, freqs_sin)
actual = fused_rope.try_fused_qk_rotary_emb(query, key, freqs_cos, freqs_sin)
assert actual is not None
actual_query, actual_key = actual
assert torch.equal(actual_query, expected_query)
assert torch.equal(actual_key, expected_key)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
def test_launch_failure_disables_runtime_key(monkeypatch):
from vllm_omni.diffusion.models.ernie_image import fused_rope
query, key, freqs_cos, freqs_sin = _inputs((1, 17, 3, 128))
launch_count = 0
def failing_launch(*args):
nonlocal launch_count
launch_count += 1
raise RuntimeError("injected launch failure")
monkeypatch.setattr(
fused_rope,
"_launch_fused_qk_rotary_emb",
failing_launch,
)
with torch.inference_mode():
for _ in range(2):
assert fused_rope.try_fused_qk_rotary_emb(query, key, freqs_cos, freqs_sin) is None
assert launch_count == 1
assert len(fused_rope._FAILED_KEYS) == 1
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
def test_failed_runtime_key_cache_is_bounded():
from vllm_omni.diffusion.models.ernie_image import fused_rope
for index in range(fused_rope._FAILED_KEYS_MAX_SIZE + 1):
fused_rope._record_failed_runtime_key((index,))
assert len(fused_rope._FAILED_KEYS) == fused_rope._FAILED_KEYS_MAX_SIZE
assert (0,) not in fused_rope._FAILED_KEYS
assert (fused_rope._FAILED_KEYS_MAX_SIZE,) in fused_rope._FAILED_KEYS
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
def test_compile_path_does_not_launch_fused_kernel(monkeypatch):
from vllm_omni.diffusion.models.ernie_image import fused_rope
query, key, freqs_cos, freqs_sin = _inputs((1, 17, 3, 128))
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
def unexpected_launch(*args):
raise AssertionError("compile path must remain native")
monkeypatch.setattr(
fused_rope,
"_launch_fused_qk_rotary_emb",
unexpected_launch,
)
with torch.inference_mode():
assert fused_rope.try_fused_qk_rotary_emb(query, key, freqs_cos, freqs_sin) is None
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
@pytest.mark.parametrize("requires_grad_input", ["query", "key", "cos", "sin"])
def test_gradient_inputs_do_not_launch_fused_kernel(monkeypatch, requires_grad_input):
from vllm_omni.diffusion.models.ernie_image import fused_rope
values = list(_inputs((1, 17, 3, 128)))
input_index = {"query": 0, "key": 1, "cos": 2, "sin": 3}[requires_grad_input]
values[input_index].requires_grad_(True)
def unexpected_launch(*args):
raise AssertionError("autograd inputs must remain on the native path")
monkeypatch.setattr(
fused_rope,
"_launch_fused_qk_rotary_emb",
unexpected_launch,
)
assert (
fused_rope.try_fused_qk_rotary_emb(
*values,
)
is None
)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
def test_mismatched_key_dtype_or_device_does_not_launch(monkeypatch):
from vllm_omni.diffusion.models.ernie_image import fused_rope
query, key, freqs_cos, freqs_sin = _inputs((1, 17, 3, 128))
def unexpected_launch(*args):
raise AssertionError("mismatched Q/K inputs must remain native")
monkeypatch.setattr(
fused_rope,
"_launch_fused_qk_rotary_emb",
unexpected_launch,
)
for unsupported_key in (key.float(), key.cpu()):
assert (
fused_rope.try_fused_qk_rotary_emb(
query,
unsupported_key,
freqs_cos,
freqs_sin,
)
is None
)
@@ -1,8 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from vllm_omni.diffusion.models.ernie_image.ernie_image_transformer import (
ErnieImageEmbedND3,
_apply_qk_rotary_emb,
_apply_rotary_emb,
rope,
)
@@ -185,3 +189,30 @@ class TestErnieImageRopePositionEmbedding:
out_diffusers = x * cos_ + rotated * sin_
assert torch.allclose(out_vllm, out_diffusers, atol=1e-6), "vllm-omni and diffusers outputs should match"
def test_apply_qk_rotary_emb_uses_native_fallback(self, monkeypatch):
"""Unsupported routes preserve both existing eager calls exactly."""
from vllm_omni.diffusion.models.ernie_image import (
ernie_image_transformer,
)
B, S, H, D = 2, 17, 3, 128
query = torch.randn(B, S, H, D)
key = torch.randn(B, S, H, D)
freqs_cos = torch.randn(B, S, D // 2)
freqs_sin = torch.randn(B, S, D // 2)
monkeypatch.setattr(
ernie_image_transformer,
"try_fused_qk_rotary_emb",
lambda *args: None,
)
actual_query, actual_key = _apply_qk_rotary_emb(query, key, freqs_cos, freqs_sin)
assert torch.equal(
actual_query,
_apply_rotary_emb(query, freqs_cos, freqs_sin),
)
assert torch.equal(
actual_key,
_apply_rotary_emb(key, freqs_cos, freqs_sin),
)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
@@ -362,6 +362,20 @@ def test_prepared_model_inputs_match_local_tokenization(
local_inputs = pipeline.prepare_model_inputs(**common_kwargs)
prepared_inputs = pipeline.prepare_model_inputs(**common_kwargs, prepared_layout=prepared_layout)
if with_reference_image:
# Preparation runs without a forward context. Coverage must come from
# the scheduler metadata copied onto this request, including every CFG row.
reused = pipeline.prepare_model_inputs(
**common_kwargs, prepared_layout=prepared_layout, kv_computed_tokens=(5,) * len(prefix_lens)
)
assert reused["cond_vae_images"] is None and reused["cond_vit_images"] is None
partial = pipeline.prepare_model_inputs(
**common_kwargs,
prepared_layout=prepared_layout,
kv_computed_tokens=(5,) * (len(prefix_lens) - 1) + (4,),
)
assert partial["cond_vae_images"] is not None and partial["cond_vit_images"] is not None
comparable_fields = (
"input_ids",
"position_ids",
@@ -474,3 +488,33 @@ def test_paged_preprocess_attaches_layout_and_scheduler_kv_requests(monkeypatch)
(12, 17, 32)
]
assert len(tokenizer.calls) == 1
@pytest.mark.parametrize("transfer_tokens, expected", [(9, [7, 5]), (4, [4, 4])])
def test_cfg_rows_preserve_reusable_token_ids_and_native_transfer_boundary(transfer_tokens, expected):
from vllm_omni.diffusion.diffusion_kv.kv_connector import prepare_kv_requests
tokenizer, image_processor = _components([12, 14])
request = _request(guidance_scale=5.0)
layout = _prepare(request, tokenizer, image_processor)
layout.tokenizer_output.think_recaption_end_pos = [[7], [5]]
layout.tokenizer_output.uncond_cfg_start_pos = [[None], [5]]
rows = build_hunyuan_diffusion_kv_requests(request, layout)
assert [row.prompt_token_ids for row in rows] == [
layout.tokenizer_output.tokens[0, :7].tolist(),
layout.tokenizer_output.tokens[1, :5].tolist(),
]
prepare_kv_requests(rows, {"num_transfer_tokens": transfer_tokens})
assert [row.num_prompt_tokens for row in rows] == expected
assert all(row.kv_transfer_params["num_transfer_tokens"] == transfer_tokens for row in rows)
layout.tokenizer_output.uncond_cfg_start_pos = [[5]]
with pytest.raises(ValueError, match="one boundary per CFG row"):
build_hunyuan_diffusion_kv_requests(request, layout)
@pytest.mark.parametrize("computed, expected", [((8, 8), True), ((8, 5), False), ((0, 0), False), ((), False)])
def test_native_image_reuse_requires_coverage_in_every_cfg_row(computed, expected):
from vllm_omni.diffusion.models.hunyuan_image3.request_layout import native_kv_covers_cond_images
output = TokenizerEncodeOutput(joint_image_slices=[[slice(2, 8)], [slice(2, 6)]])
assert native_kv_covers_cond_images(output, computed) is expected
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for HunyuanImage3 AR sampler logic (stage transitions,
ratio restriction, comprehension blocking)."""
@@ -21,6 +21,10 @@ RATIO_END = 210
RATIO_OTHER_START = 220
RATIO_OTHER_END = 223
TIMESTEP = 224
CFG = 110
IMG_TOK = 111
JOINT_SEP = 112
EOI = 113
class FakeSamplerModel:
@@ -47,10 +51,13 @@ class FakeSamplerModel:
self._stage_transitions[END_OF_THINK] = [RECAPTION]
self._stage_transitions[END_OF_RECAPTION] = [ANSWER, BOI, SIZE_TOKEN]
self._blocked_token_ids: set[int] = set()
if is_comprehension:
self._blocked_token_ids.update([BOI, SIZE_TOKEN])
self._blocked_token_ids.update(self._all_ratio_ids)
# Mirrors the model: the blocklist is always built so generation-mode
# engines can apply it to per-request comprehension rows (#6088).
self._cfg_token_id = CFG
self._blocked_token_ids: set[int] = {BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP}
self._blocked_token_ids.update(self._all_ratio_ids)
self._blocked_token_ids_tensor = None
self._sampler = None
# Bind the real methods from the model class.
from vllm_omni.model_executor.models.hunyuan_image3.hunyuan_image3 import (
@@ -59,6 +66,8 @@ class FakeSamplerModel:
_get_forced_token = _Real._get_forced_token
_apply_ratio_restriction = _Real._apply_ratio_restriction
_row_is_comprehension = staticmethod(_Real._row_is_comprehension)
sample = _Real.sample
class FakeTokenEmbedding:
@@ -250,3 +259,114 @@ class TestForceEosAfterRatio:
assert logits[0, EOS].item() == 0
non_eos_max = logits[0, :EOS].max().item()
assert non_eos_max == min_score
class _RecordingSampler:
"""Stub sampler: returns the (mutated) logits so tests can assert on them."""
def __call__(self, *, logits, sampling_metadata):
return logits
def _metadata(output_token_ids):
from types import SimpleNamespace
return SimpleNamespace(output_token_ids=output_token_ids)
class TestPerRequestTaskMode:
"""Generation-mode engines must honor per-request comprehension rows (#6088).
A text chat (I2T/T2T) served by an AR+DiT deployment is marked with
extra_args={"ar_task_mode": "comprehension"}; its rows get the stateless
image-token blocking instead of stage-transition forcing, while untagged
rows keep the generation behavior.
"""
VOCAB = 300
def _run(self, model, histories, per_req_extra_args):
logits = torch.zeros(len(histories), self.VOCAB)
logits[:, 50] = 7.0 # ordinary text token, must survive comprehension
model._sampler = _RecordingSampler()
out = model.sample(logits, _metadata(histories), per_req_extra_args=per_req_extra_args)
return out
def test_comprehension_row_skips_forcing_and_blocks_image_tokens(self):
model = FakeSamplerModel(is_comprehension=False)
min_score = torch.finfo(torch.float32).min
logits = self._run(
model,
histories=[[END_OF_THINK], [END_OF_THINK]],
per_req_extra_args=[{"ar_task_mode": "comprehension"}, None],
)
# Row 0 (comprehension): no </think> -> <recaption> forcing (ordinary
# text token survives, which a force-fill would have erased) and every
# DiT scaffold token is masked.
assert logits[0, 50].item() == 7.0
for tid in (BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP, RATIO_START, RATIO_OTHER_START):
assert logits[0, tid].item() == min_score, f"token {tid} not blocked"
# Row 1 (generation): forced to <recaption> exactly as before.
assert logits[1, RECAPTION].item() == 0
assert logits[1, 50].item() == min_score
def test_no_extra_args_keeps_generation_behavior(self):
model = FakeSamplerModel(is_comprehension=False)
min_score = torch.finfo(torch.float32).min
logits = self._run(model, histories=[[END_OF_THINK]], per_req_extra_args=None)
assert logits[0, RECAPTION].item() == 0
assert logits[0, 50].item() == min_score
def test_empty_or_foreign_extra_args_keeps_generation_behavior(self):
model = FakeSamplerModel(is_comprehension=False)
logits = self._run(
model,
histories=[[END_OF_THINK], [END_OF_THINK]],
per_req_extra_args=[{}, {"other_key": 1}],
)
assert logits[0, RECAPTION].item() == 0
assert logits[1, RECAPTION].item() == 0
def test_comprehension_row_without_trigger_history(self):
model = FakeSamplerModel(is_comprehension=False)
min_score = torch.finfo(torch.float32).min
logits = self._run(
model,
histories=[[1, 2, 3]],
per_req_extra_args=[{"ar_task_mode": "comprehension"}],
)
assert logits[0, 50].item() == 7.0
assert logits[0, CFG].item() == min_score
def test_static_comprehension_engine_blocks_cfg(self):
model = FakeSamplerModel(is_comprehension=True)
min_score = torch.finfo(torch.float32).min
logits = self._run(model, histories=[[1, 2]], per_req_extra_args=None)
for tid in (BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP):
assert logits[0, tid].item() == min_score
assert logits[0, 50].item() == 7.0
def test_extra_args_shorter_than_batch_defaults_to_generation(self):
model = FakeSamplerModel(is_comprehension=False)
logits = self._run(
model,
histories=[[END_OF_THINK], [END_OF_THINK]],
per_req_extra_args=[{"ar_task_mode": "comprehension"}],
)
min_score = torch.finfo(torch.float32).min
assert logits[0, 50].item() == 7.0 # tagged row: comprehension
assert logits[1, RECAPTION].item() == 0 # missing entry: generation
assert logits[1, 50].item() == min_score
@@ -9,6 +9,7 @@ from __future__ import annotations
import math
from contextlib import ExitStack, contextmanager
from types import SimpleNamespace
from unittest.mock import patch
import pytest
@@ -143,6 +144,104 @@ def test_cache_manager_registers_attention_without_adding_dense_state() -> None:
assert mgr.state_dict() == {}
def test_cache_manager_initializes_with_quantization_skip_layers(monkeypatch):
import vllm_omni.diffusion.attention.layer as layer
from vllm_omni.diffusion.config import set_current_diffusion_config
from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_image3_transformer import ImageKVCacheManager
config = SimpleNamespace(
diffusion_attention_config=None,
parallel_config=SimpleNamespace(ring_degree=1),
diffusion_kv_cache_dtype="auto",
diffusion_kv_cache_skip_layer_indices={3},
)
monkeypatch.setattr(layer, "get_attn_backend_for_role", lambda **_: (layer.SDPABackend, None))
monkeypatch.setattr(layer, "build_parallel_attention_strategy", lambda **_: object())
with patched_mgr_env(), set_current_diffusion_config(config):
# Keep the real Attention constructor and its skip-layer validation.
with patch(f"{_TRANSFORMER_MODULE}.Attention", layer.Attention):
mgr = ImageKVCacheManager(
num_heads=NUM_HEADS,
num_kv_heads=NUM_KV_HEADS,
head_dim=HEAD_DIM,
scaling=SCALING,
prefix="model.layers.3.self_attn",
)
assert mgr.attn.layer_idx == 3
assert not mgr.attn._should_apply_kv_cache_quant()
@pytest.mark.parametrize("first_step", [True, False])
def test_native_connector_keeps_imported_prefix_in_paged_attention(first_step) -> None:
from vllm_omni.diffusion.forward_context import override_paged_kv_adapter, set_forward_context
mgr = _make_cache_mgr()
mgr.attn.paged_kv_active = True
query_len, prefix_len = IMAGE_TOKEN_LEN, 19
key, value = _make_known_kv(2 * query_len)
spans = [[(prefix_len, prefix_len + query_len)]] * 2
with (
set_forward_context(),
override_paged_kv_adapter(object()),
patch.object(mgr, "_forward_dense_legacy", side_effect=AssertionError("dense KV path")),
patch(f"{_TRANSFORMER_MODULE}.repeat_kv", side_effect=AssertionError("KV head expansion")),
patch(f"{_TRANSFORMER_MODULE}.get_paged_kv_computed_tokens", return_value=(prefix_len, prefix_len)),
):
_call_mgr(
mgr,
2,
query_len,
prefix_len + query_len,
key,
value,
first_step=first_step,
full_attn_spans=spans,
)
assert len(mgr.attn.paged_calls) == 1
assert mgr.attn.paged_calls[0][1].shape == (2, query_len, NUM_KV_HEADS, HEAD_DIM)
assert mgr.attn.paged_metadata[0].attn_mask is None
assert mgr.attn.paged_metadata[0].full_attn_spans == spans
assert mgr.image_kv_cache_map is None
assert not hasattr(mgr, "_materialize_paged_prefix")
def test_legacy_tensor_prefix_still_uses_dense_cache_and_reuse() -> None:
mgr = _make_cache_mgr()
prefix_len, prompt_len = 3, 2
imported = _make_known_kv(prefix_len, base=10.0)
mgr._injected_ar_kv = [imported]
first_len = prompt_len + IMAGE_TOKEN_LEN
key, value = _make_known_kv(first_len, base=20.0)
with (
patch.object(mgr, "_forward_paged", side_effect=AssertionError("legacy entered paged path")),
):
_call_mgr(
mgr,
1,
first_len,
prefix_len + first_len,
key,
value,
first_step=True,
gen_timestep_scatter_index=_gen_timestep_index(1, prompt_len),
)
assert mgr.image_kv_cache_map is not None
assert mgr.image_kv_cache_map[0].shape[1] == prefix_len + prompt_len
next_key, next_value = _make_known_kv(IMAGE_TOKEN_LEN, base=30.0)
_call_mgr(
mgr,
1,
IMAGE_TOKEN_LEN,
prefix_len + first_len,
next_key,
next_value,
position_ids=torch.arange(prefix_len + prompt_len, prefix_len + first_len).unsqueeze(0),
)
assert len(mgr.attn.calls) == 2
assert not mgr.attn.paged_calls
assert mgr.attn.calls[-1][1].shape == (1, prefix_len + first_len, NUM_HEADS, HEAD_DIM)
def test_scheduler_paged_kv_runs_piecewise_for_first_and_later_steps() -> None:
from vllm_omni.diffusion.forward_context import override_paged_kv_adapter, set_forward_context
@@ -12,6 +12,7 @@ from vllm_omni.diffusion.models.lingbot_world.actions import (
LINGBOT_CAMERA_ACTION_SCHEMA,
LINGBOT_CAMERA_TRAJECTORY_SCHEMA,
LingBotCameraControlReducer,
camera_trajectory_from_absolute_pose,
integrate_lingbot_camera_actions,
parse_lingbot_camera_action_frames,
parse_lingbot_camera_action_script,
@@ -212,6 +213,11 @@ def test_action_integrator_matches_lingbot_motion_and_calibration() -> None:
torch.tensor([500.0, 500.0, 208.0, 120.0]),
)
with pytest.raises(ValueError, match="resolution must be positive"):
camera_trajectory_from_absolute_pose(trajectory.poses[:1], width=0, height=480)
with pytest.raises(ValueError, match=r"\[frames, 4, 4\]"):
camera_trajectory_from_absolute_pose(torch.zeros(4, 4), width=832, height=480)
@pytest.mark.parametrize(
("data", "message"),
@@ -74,6 +74,17 @@ def _install_vllm_stubs() -> None:
_SeqAllToAll4D,
)
def _all_to_all_5D(value, scatter_idx=3, gather_idx=1, group=None, use_sync=False):
# Single-rank stub: the fused q/k/v exchange is the identity when N == 1.
del scatter_idx, gather_idx, group, use_sync
return value
setattr(
sys.modules["vllm_omni.diffusion.distributed.comm"],
"all_to_all_5D",
_all_to_all_5D,
)
def get_sp_group():
return SimpleNamespace(
ulysses_world_size=1,
@@ -588,26 +599,7 @@ def test_tp_rmsnorm_weight_loader_selects_rank_shard(monkeypatch: pytest.MonkeyP
torch.testing.assert_close(norm.weight, torch.tensor([30.0, 40.0]))
def test_single_token_text_kv_shard_owns_compact_storage(monkeypatch):
module = _load_module()
monkeypatch.setattr(
module,
"get_sp_group",
lambda: SimpleNamespace(
ulysses_world_size=2,
ulysses_rank=1,
ulysses_group=None,
),
)
attention = module.LingBotCrossAttention(dim=8, num_heads=4)
full = torch.arange(8, dtype=torch.float32).reshape(1, 1, 4, 2)
shard = attention.shard_kv_heads(full)
torch.testing.assert_close(shard, full[:, :, 2:], rtol=0, atol=0)
assert shard.is_contiguous()
assert shard.untyped_storage().nbytes() == shard.numel() * shard.element_size()
@pytest.mark.parametrize("attention_class", ["LingBotSelfAttention", "LingBotCrossAttention"])
@pytest.mark.parametrize("attention_class", ["LingBotSelfAttention"])
def test_ulysses_rejects_non_divisible_head_count(monkeypatch, attention_class):
module = _load_module()
monkeypatch.setattr(
@@ -621,3 +613,35 @@ def test_ulysses_rejects_non_divisible_head_count(monkeypatch, attention_class):
)
with pytest.raises(ValueError, match="heads must be divisible"):
getattr(module, attention_class)(dim=8, num_heads=4)
def test_supplied_camera_modulation_matches_computing_it_inline() -> None:
"""Passing a prebuilt modulation must be arithmetically identical to letting the block build its own."""
module = _load_module()
torch.manual_seed(0)
block = module.LingBotAttentionBlock(dim=4, num_heads=2, prefix="blocks.0")
camera = torch.randn(1, 3, 4)
scale, shift = block.camera_modulation(camera)
inline_scale, inline_shift = block.camera_modulation(camera)
torch.testing.assert_close(scale, inline_scale, rtol=0, atol=0)
torch.testing.assert_close(shift, inline_shift, rtol=0, atol=0)
def test_the_camera_modulation_reads_only_the_camera_tokens() -> None:
"""The reuse window is only sound because the modulation depends on nothing else that varies per forward."""
module = _load_module()
torch.manual_seed(0)
block = module.LingBotAttentionBlock(dim=4, num_heads=2, prefix="blocks.0")
camera = torch.randn(1, 3, 4)
first = block.camera_modulation(camera)
# Same camera, different everything else a forward would carry: the modulation cannot move.
second = block.camera_modulation(camera.clone())
torch.testing.assert_close(first[0], second[0], rtol=0, atol=0)
torch.testing.assert_close(first[1], second[1], rtol=0, atol=0)
# A different camera must move it, or the cache key would be hiding a real dependency.
other = block.camera_modulation(camera + 1)
assert not torch.equal(first[0], other[0])
@@ -422,7 +422,17 @@ def test_build_plucker_embedding_uses_framewise_deltas_for_rotations_and_transla
torch.testing.assert_close(result[2, 3:, 0, 1], expected_direction)
def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm() -> None:
@pytest.mark.parametrize(
("translation_scale", "expected_origins"),
[
(None, [[0.0, 0.0, 0.0], [0.3, 0.4, 0.0], [0.6, 0.8, 0.0]]),
(0.05, [[0.0, 0.0, 0.0], [60.0, 80.0, 0.0], [120.0, 160.0, 0.0]]),
],
)
def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm(
translation_scale: float | None,
expected_origins: list[list[float]],
) -> None:
poses = torch.eye(4).repeat(3, 1, 1)
poses[:, :3, 3] = torch.tensor(
[
@@ -440,10 +450,37 @@ def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm()
target_width=1,
device=torch.device("cpu"),
dtype=torch.float32,
translation_scale=translation_scale,
)
expected_origins = torch.tensor([[0.0, 0.0, 0.0], [0.3, 0.4, 0.0], [0.6, 0.8, 0.0]])
torch.testing.assert_close(result[:, :3, 0, 0], expected_origins)
torch.testing.assert_close(result[:, :3, 0, 0], torch.tensor(expected_origins))
from vllm_omni.diffusion.models.lingbot_world.actions import LINGBOT_CONTROLLER_TRANSLATION_UNIT
if translation_scale != LINGBOT_CONTROLLER_TRANSLATION_UNIT:
return
# Fixed controller unit keeps relative speed; max-norm would collapse both to 1.
slow = torch.eye(4).repeat(2, 1, 1)
slow[1, 2, 3] = 0.005
fast = torch.eye(4).repeat(2, 1, 1)
fast[1, 2, 3] = 0.5
kwargs = dict(
height=1,
width=1,
target_height=1,
target_width=1,
device=torch.device("cpu"),
dtype=torch.float32,
translation_scale=LINGBOT_CONTROLLER_TRANSLATION_UNIT,
)
slow_origin = build_plucker_embedding(_trajectory(slow), **kwargs)[1, :3, 0, 0]
fast_origin = build_plucker_embedding(_trajectory(fast), **kwargs)[1, :3, 0, 0]
torch.testing.assert_close(slow_origin, torch.tensor([0.0, 0.0, 0.1]))
torch.testing.assert_close(fast_origin, torch.tensor([0.0, 0.0, 10.0]))
wasd = torch.eye(4).repeat(2, 1, 1)
wasd[1, 2, 3] = LINGBOT_CONTROLLER_TRANSLATION_UNIT
wasd_origin = build_plucker_embedding(_trajectory(wasd), **kwargs)[1, :3, 0, 0]
torch.testing.assert_close(wasd_origin, torch.tensor([0.0, 0.0, 1.0]))
def test_build_plucker_embedding_interpolates_raw_poses_before_framewise_conversion() -> None:
@@ -837,6 +837,9 @@ def _rollout(model, mode, dtype, batch):
frames_per_block=_FRAMES,
max_scratch_tokens_per_branch=_FRAMES * _TOKENS_PER_FRAME,
cross_attention_lengths={"text": 5},
# Cross-attention keeps every local head on every SP rank, so its pool is wider than the
# self-attention share above.
cross_attention_kv_heads={"text": model.blocks[0].cross_attn.num_local_heads},
device=device,
)
state = ARDiffusionKVState(kv, "numeric", {"main": kv.begin_request("numeric")}, num_layers=_LAYERS)
@@ -896,7 +899,6 @@ def _worker(rank, world_size, sp_size, tp_size, mode, dtype, batch, rendezvous):
from vllm_omni.diffusion.distributed.parallel_state import (
destroy_distributed_env,
destroy_model_parallel,
get_sp_group,
init_distributed_environment,
initialize_model_parallel,
)
@@ -948,11 +950,10 @@ def _worker(rank, world_size, sp_size, tp_size, mode, dtype, batch, rendezvous):
bound = 1e-5 if dtype == torch.float32 else 1e-2
error = (result - expected).double().norm() / expected.double().norm()
assert error <= bound, f"rank={rank}, TP{tp} SP{sp}: relative L2 {error.item():.3g} > {bound}"
local_heads = _HEADS // tp // sp
first = (
get_tensor_model_parallel_rank() * (_HEADS // tp)
+ get_sp_group().ulysses_rank * local_heads
)
# Cross-attention K/V is replicated across the Ulysses group, not split over it, so
# every SP rank holds its whole TP shard of heads. Only TP slices the cache now.
local_heads = _HEADS // tp
first = get_tensor_model_parallel_rank() * local_heads
for (k, v), (ek, ev) in zip(cross, expected_cross, strict=True):
tolerance = 1e-5 if dtype == torch.float32 else 2e-2
torch.testing.assert_close(
@@ -1033,3 +1034,56 @@ def test_temporal_rope_extension_keeps_spatial_limit():
model._rotary_embedding(
frames=3, height=17, width=1, start_frame=16, dtype=torch.float32, device=torch.device("cpu")
)
@pytest.mark.cpu
def test_the_camera_injector_runs_once_per_held_cache_and_gives_the_same_output() -> None:
"""Two forwards sharing one caller-held cache: the injector runs once per block, outputs unchanged."""
module = attention_tests._load_module()
model = _tiny_model(module, num_layers=2).eval()
hidden_states = torch.randn(1, 36, 1, 4, 4)
timestep = torch.tensor([1.0])
encoder_hidden_states = torch.randn(1, 3, 6)
camera_hidden_states = torch.randn(1, 6 * 8 * 8, 1, 4, 4)
def run(cache, camera_cache=None):
return model(
hidden_states,
timestep,
encoder_hidden_states,
camera_hidden_states,
cache=cache,
start_frame=0,
update_cache=False,
camera_modulation_cache=camera_cache,
)
# Baseline: no cache, so every forward rebuilds the injector in every block.
uncached_first = run(_cache(module, model))
uncached_second = run(_cache(module, model))
injector_calls = [attention_tests._record_outputs(block.cam_injector_layer1) for block in model.blocks]
held = module.CameraModulationCache()
cached_first = run(_cache(module, model), held)
cached_second = run(_cache(module, model), held)
# One build per block for both forwards, instead of one per block per forward.
assert [len(calls) for calls in injector_calls] == [1, 1]
torch.testing.assert_close(cached_first, uncached_first, rtol=0, atol=0)
torch.testing.assert_close(cached_second, uncached_second, rtol=0, atol=0)
@pytest.mark.cpu
def test_timestep_projection_is_staged_into_one_static_buffer_per_shape() -> None:
"""Two forwards' projections land in the same buffer; the blocks never see the fresh per-forward tensor."""
module = attention_tests._load_module()
model = _tiny_model(module, num_layers=1).eval()
first = torch.randn(1, 8, 6, model.dim)
second = torch.randn(1, 8, 6, model.dim)
staged_first = model._stage_timestep_projection(first)
assert staged_first is not first and torch.equal(staged_first, first)
staged_second = model._stage_timestep_projection(second)
assert staged_second is staged_first and torch.equal(staged_second, second)
# A different shape gets its own buffer; the first one is kept.
other = model._stage_timestep_projection(torch.randn(1, 4, 6, model.dim))
assert other is not staged_first and len(model._timestep_projection_buffers) == 2
@@ -4,11 +4,13 @@
from __future__ import annotations
import gc
import os
import weakref
from pathlib import Path
from threading import Lock
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from diffusers.utils.torch_utils import randn_tensor as _diffusers_randn_tensor
@@ -18,6 +20,7 @@ from torch import nn
import vllm_omni.diffusion.models.lingbot_world.dmd_block as lingbot_dmd_block
import vllm_omni.diffusion.models.lingbot_world.pipeline as lingbot_pipeline
from tests.diffusion.models.wan2_2.conftest import noop_progress_bar
from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession
from vllm_omni.diffusion.models.interface import SupportsStepExecution, supports_step_execution
from vllm_omni.diffusion.models.lingbot_world.actions import (
integrate_lingbot_camera_actions,
@@ -28,6 +31,7 @@ from vllm_omni.diffusion.models.lingbot_world.camera import (
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.utils import StepRequestState
from vllm_omni.experimental.ar_diffusion.capability import supports_chunk_step_grouping
from vllm_omni.experimental.ar_diffusion.tick_protocol import (
ARDiffusionControlInput,
ARDiffusionTickRequest,
@@ -126,6 +130,10 @@ class _RecordingTransformer(nn.Module):
sink_size=3,
)
self.blocks = nn.ModuleList([nn.Identity(), nn.Identity()])
for block in self.blocks:
# The spec reads head geometry from both attentions; cross-attention keeps every local head.
block.self_attn = SimpleNamespace(num_sp_heads=2)
block.cross_attn = SimpleNamespace(num_local_heads=2)
self.calls: list[dict] = []
self.cache_allocations: list[dict] = []
self.raise_on_call = raise_on_call
@@ -145,6 +153,7 @@ class _RecordingTransformer(nn.Module):
"cache_id": id(kwargs["cache"]),
"start_frame": kwargs["start_frame"],
"update_cache": kwargs["update_cache"],
"camera_cache": kwargs.get("camera_modulation_cache"),
}
self.calls.append(call)
if self.raise_on_call == len(self.calls):
@@ -411,8 +420,10 @@ def _stub_pipeline_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:
intrinsics=value.intrinsics[:num_frames],
)
def build_plucker_embedding(value, *, height, width, target_height, target_width, device, dtype):
del target_height, target_width
def build_plucker_embedding(
value, *, height, width, target_height, target_width, device, dtype, translation_scale=None
):
del target_height, target_width, translation_scale
frames = value.poses.shape[0]
data = torch.arange(frames * 6 * height * width, device=device, dtype=torch.float32)
return data.reshape(frames, 6, height, width).to(dtype=dtype)
@@ -531,14 +542,15 @@ def test_ar_diffusion_capability_uses_transformer_local_head_geometry() -> None:
assert spec.sink_frames == 3
assert [(branch.name, branch.local_index) for branch in spec.kv_branches] == [("main", 0)]
assert spec.cross_attention_lengths == {"text": 512}
# One condition block, committed + pending encoder caches, and decoder state.
# One condition block, committed + pending encoder caches, the session's
# cached prompt embedding (512 tokens x text_dim 8 x fp32), and decoder state.
# The stub encoder retains 2 RGB frames at 16x16 and 2 feature frames at 2x2.
encoder_bytes = (2 * 3 * 16 * 16 + 2 * 16 * 2 * 2) * 4
assert spec.model_owned_state_bytes_per_session == 960 + 2 * encoder_bytes + 2_421_248
assert spec.model_owned_state_bytes_per_session == 960 + 384 + 2 * encoder_bytes + 2_421_248
def test_session_admission_accounts_for_the_streaming_decoder_cache() -> None:
"""Admission includes encoder state even when the decoder cannot stream."""
"""Admission includes encoder state and the prompt embedding even when the decoder cannot stream."""
module = _load_pipeline_module()
def spec_pipeline():
@@ -557,13 +569,29 @@ def test_session_admission_accounts_for_the_streaming_decoder_cache() -> None:
# than the image condition alone could account for.
assert widened - small > 2_000_000
# Removing decoder support leaves the condition and both encoder histories.
# Removing decoder support leaves the condition, both encoder histories and
# the prompt-embedding cache.
bare = spec_pipeline()
for attribute in ("decoder", "post_quant_conv"):
if hasattr(bare.vae, attribute):
delattr(bare.vae, attribute)
bare.vae._cached_conv_counts.pop("decoder")
assert bare.ar_diffusion_kv_cache_spec().model_owned_state_bytes_per_session == 960 + 2 * 6_656
assert bare.ar_diffusion_kv_cache_spec().model_owned_state_bytes_per_session == 960 + 384 + 2 * 6_656
@pytest.mark.parametrize("sp_size", [1, 4])
def test_session_admission_budgets_camera_modulation_per_rank(sp_size: int) -> None:
pipeline = _pipeline(_load_pipeline_module())
pipeline._ar_height, pipeline._ar_width = 480, 832
pipeline.od_config.parallel_config.ulysses_degree = sp_size
pipeline.transformer.config.num_layers = 40
pipeline.transformer.config.num_attention_heads = 40
pipeline.transformer.config.attention_head_dim = 128
pipeline.transformer._dtype = torch.bfloat16
forty_layers = pipeline.ar_diffusion_kv_cache_spec().model_owned_state_bytes_per_session
pipeline.transformer.config.num_layers = 1
one_layer = pipeline.ar_diffusion_kv_cache_spec().model_owned_state_bytes_per_session
assert forty_layers - one_layer == 2 * 39 * (4680 // sp_size) * 5120 * 2
def test_preprocess_materializes_external_inputs_before_worker_execution(tmp_path: Path) -> None:
@@ -1330,10 +1358,34 @@ def test_first_frame_condition_and_camera_fold_match_transformer_contract() -> N
assert expected_camera.shape == (1, 384, 3, 2, 2)
def test_fixed_dmd_transition_and_cache_commit_trace() -> None:
@pytest.mark.parametrize("value", ["true", "false", 1, 0, None])
def test_reuse_last_step_kv_rejects_non_boolean_config(value: object) -> None:
config = _od_config()
config.model_config["lingbot_reuse_last_step_kv"] = value
with pytest.raises(ValueError, match="lingbot_reuse_last_step_kv must be a bool"):
_pipeline(_load_pipeline_module(), od_config=config)
def test_reuse_last_step_kv_is_fixed_per_instance(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("VLLM_OMNI_LINGBOT_REUSE_LAST_STEP_KV", "1")
module = _load_pipeline_module()
config = _od_config()
config.model_config["lingbot_reuse_last_step_kv"] = True
reuse = _pipeline(module, od_config=config)
default = _pipeline(module)
config.model_config["lingbot_reuse_last_step_kv"] = False
assert reuse._dmd_blocks.reuse_last_step_kv is True
assert default._dmd_blocks.reuse_last_step_kv is False
assert reuse._dmd_blocks is reuse._dmd_blocks
@pytest.mark.parametrize("reuse_last_step_kv", [False, True])
def test_fixed_dmd_transition_and_cache_commit_trace(reuse_last_step_kv: bool) -> None:
module = _load_pipeline_module()
transformer = _RecordingTransformer()
pipeline = _pipeline(module, transformer=transformer)
config = _od_config()
config.model_config["lingbot_reuse_last_step_kv"] = reuse_last_step_kv
pipeline = _pipeline(module, transformer=transformer, od_config=config)
result = pipeline(_request())
@@ -1355,14 +1407,18 @@ def test_fixed_dmd_transition_and_cache_commit_trace() -> None:
current = x0
torch.testing.assert_close(result.output, current)
expected_timesteps = [timestep for timestep, _ in warped_schedule]
if not reuse_last_step_kv:
expected_timesteps.append(0.0)
torch.testing.assert_close(
torch.cat([call["timestep"] for call in transformer.calls]),
torch.tensor([*(timestep for timestep, _ in warped_schedule), 0.0]),
torch.cat([call["timestep"] for call in transformer.calls]), torch.tensor(expected_timesteps)
)
assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True]
assert [call["start_frame"] for call in transformer.calls] == [0, 0, 0, 0, 0]
commit_flags = [False] * (len(expected_timesteps) - 1) + [True]
assert [call["update_cache"] for call in transformer.calls] == commit_flags
assert [call["start_frame"] for call in transformer.calls] == [0] * len(expected_timesteps)
assert len({call["cache_id"] for call in transformer.calls}) == 1
torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], result.output)
committed_latent = result.output + warped_schedule[-1][1] if reuse_last_step_kv else result.output
torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], committed_latent)
@pytest.mark.parametrize(
@@ -1669,7 +1725,6 @@ def test_typed_ticks_generate_one_global_block_and_return_standard_metadata(
"applied_event_ids": [1],
}
assert pipeline._ar_sessions["world-1"].next_chunk_index == 2
assert pipeline._ar_sessions["world-1"].prompt == "enter the snowy valley"
assert pipeline._ar_sessions["world-1"].camera_tail is not None
assert pipeline._ar_sessions["world-1"].camera_tail.poses.shape == (1, 4, 4)
expected_generator = torch.Generator(device="cpu").manual_seed(17)
@@ -1783,6 +1838,7 @@ def test_first_typed_yaw_action_uses_pre_action_identity_anchor(
target_width=16,
device=torch.device("cpu"),
dtype=torch.float32,
translation_scale=None, # None for camera_actions / action_script input (using max-norm)
)[1:]
assert not torch.equal(action_embedding, neutral_embedding)
@@ -1842,8 +1898,11 @@ def test_typed_ticks_keep_bounded_encoder_state_beyond_ten_chunks(
pipeline(_request(sampling=sampling))
assert pipeline.vae.encode_inputs == []
assert [video.shape[2] for video in pipeline.vae.encoder.inputs] == [1] + [4] * 32
# The stub encoder's history settles after its second block; the nine
# blocks after that reuse the settled condition without an encode.
assert [video.shape[2] for video in pipeline.vae.encoder.inputs] == [1] + [4] * 5
assert all(torch.count_nonzero(video) == 0 for video in pipeline.vae.encoder.inputs[1:])
assert pipeline._ar_sessions["world-1"].condition_fixed_point is not None
encoder_cache = pipeline._ar_sessions["world-1"].encoder_cache
assert sum(t.numel() * t.element_size() for t in encoder_cache) == pipeline._condition_encoder_cache_bytes()
assert pipeline.vae._enc_feat_map == ["module-owned"]
@@ -1939,6 +1998,76 @@ def test_condition_continues_real_causal_vae_beyond_initial_horizon(
assert pipeline._ar_sessions == {}
def test_condition_encoder_stops_at_its_fixed_point_and_restarts_with_the_session(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Once a block returns the history it was given, later blocks reuse the condition without encoding.
The whole-clip encode is the reference for every block, so reusing the
stored condition is only allowed to be a speedup, never a change.
"""
pipeline = _tiny_condition_pipeline(monkeypatch)
pipeline._ar_diffusion_kv_state = object()
blocks = 12
inputs = pipeline._parse_request(_request(sampling=_SamplingParams(num_frames=(blocks * 3 - 1) * 4 + 1)))
with torch.inference_mode():
expected = pipeline._prepare_condition(inputs, dtype=torch.float32)
encoder_calls = []
original_encoder = pipeline.vae.encoder.forward
def encoder(*args, **kwargs):
encoder_calls.append(len(seen))
return original_encoder(*args, **kwargs)
seen: list[torch.Tensor] = []
def generate_block(**kwargs):
index = len(seen) % blocks
torch.testing.assert_close(
kwargs["condition"], expected[:, :, index * 3 : (index + 1) * 3], rtol=1e-6, atol=1e-6
)
seen.append(kwargs["condition"])
return torch.zeros_like(kwargs["condition"][:, :16])
monkeypatch.setattr(pipeline.vae.encoder, "forward", encoder)
monkeypatch.setattr(pipeline, "_generate_block", generate_block)
def tick(index):
pipeline(_request(sampling=_SamplingParams(extra_args=_tick_extra_args(chunk_index=index))))
settled_at = None
for index in range(blocks):
tick(index)
state = pipeline._ar_sessions["world-1"]
if settled_at is None and state.condition_fixed_point is not None:
settled_at = index
assert state.pending_encoder_cache is None and state.encoder_cache is not None
assert settled_at is not None and 0 < settled_at < blocks - 1, settled_at
# Three encoder passes per encoded block, none once the fixed point is stored.
assert encoder_calls == [index for index in range(settled_at + 1) for _ in range(3)]
# The stored condition is the session's own, not shared with the callers.
assert all(seen[settled_at] is not later for later in seen[settled_at + 1 :])
# A session restarted from chunk 0 encodes again from the source image.
pipeline.reset_ar_diffusion_session("world-1")
tick(0)
state = pipeline._ar_sessions["world-1"]
assert state.condition_fixed_point is None
assert encoder_calls[-3:] == [blocks] * 3
def test_condition_encoder_fixed_point_is_released_with_the_session(monkeypatch: pytest.MonkeyPatch) -> None:
pipeline = _tiny_condition_pipeline(monkeypatch)
pipeline._ar_diffusion_kv_state = object()
monkeypatch.setattr(pipeline, "_generate_block", lambda **kwargs: torch.zeros_like(kwargs["condition"][:, :16]))
for index in range(8):
pipeline(_request(sampling=_SamplingParams(extra_args=_tick_extra_args(chunk_index=index))))
state = pipeline._ar_sessions["world-1"]
assert state.condition_fixed_point is not None
pipeline.close_ar_diffusion_session("world-1")
assert state.condition_fixed_point is None and state.encoder_cache is None
def test_condition_encoder_interleaving_retry_and_reset(monkeypatch: pytest.MonkeyPatch) -> None:
pipeline = _tiny_condition_pipeline(monkeypatch)
pipeline._ar_diffusion_kv_state = object()
@@ -2015,6 +2144,7 @@ def test_stepwise_failed_chunk_releases_pending_encoder_on_close(monkeypatch: py
transformer.raise_on_call = 1
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
pipeline.prepare_encode(state)
pipeline.prepare_next_chunk(state)
session = pipeline._ar_sessions[state.request_id]
assert session.encoder_cache is None and session.pending_encoder_cache is not None
with pytest.raises(RuntimeError, match="forced transformer failure"):
@@ -2037,6 +2167,7 @@ def test_stateful_condition_rejects_tiled_encoder(monkeypatch: pytest.MonkeyPatc
state = _stepwise_state()
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
pipeline.prepare_encode(state)
pipeline.prepare_next_chunk(state)
assert pipeline.vae.encoder.inputs == []
assert all(s.encoder_cache is None and s.pending_encoder_cache is None for s in pipeline._ar_sessions.values())
@@ -2168,11 +2299,14 @@ def _stepwise_state(
def _run_stepwise(pipeline, state):
outputs = []
pipeline.prepare_encode(state)
pipeline.prepare_next_chunk(state)
while not state.request_denoise_completed:
noise = pipeline.denoise_step(None, states=[state])
pipeline.step_scheduler(state, noise)
if state.chunk_denoise_completed:
outputs.append(pipeline.post_decode(state))
if not state.request_denoise_completed:
pipeline.prepare_next_chunk(state)
return outputs
@@ -2186,6 +2320,7 @@ def _stepwise_chunks(pipeline, state, ar_state):
"""
with pipeline.bind_ar_diffusion_state(state.request_id, ar_state):
pipeline.prepare_encode(state)
pipeline.prepare_next_chunk(state)
while not state.request_denoise_completed:
output = None
with pipeline.bind_ar_diffusion_state(state.request_id, ar_state):
@@ -2193,10 +2328,46 @@ def _stepwise_chunks(pipeline, state, ar_state):
pipeline.step_scheduler(state, noise)
if state.chunk_denoise_completed:
output = pipeline.post_decode(state)
if not state.request_denoise_completed:
pipeline.prepare_next_chunk(state)
if output is not None:
yield output
def test_stepwise_block_shares_one_camera_cache_across_its_five_forwards() -> None:
"""The four probes and the commit of one stepwise block reuse one camera cache; the next block gets its own.
Request mode creates one cache per loop; the stepwise path's forwards are
separate scheduler steps, so the pipeline holds the block's cache and
passes it to each one.
"""
module = _load_pipeline_module()
transformer = _RecordingTransformer()
pipeline = _pipeline(module, transformer=transformer)
pipeline._ar_height = 16
pipeline._ar_width = 16
state = _stepwise_state(num_frames=21)
state.sampling.output_type = "latent"
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
outputs = _run_stepwise(pipeline, state)
assert len(outputs) == 2 and len(transformer.calls) == 10
caches = [call["camera_cache"] for call in transformer.calls]
assert all(c is not None for c in caches)
assert len({id(c) for c in caches[:5]}) == 1 and len({id(c) for c in caches[5:]}) == 1
assert caches[0] is not caches[5]
# The block's cache does not outlive its commit.
assert "camera_cache" not in state.extra
def test_pipeline_declares_chunk_step_grouping() -> None:
"""A realtime block's probes and commit may run without a scheduler cycle between them."""
module = _load_pipeline_module()
assert module.LingBotWorldCausalDMDPipeline.supports_chunk_step_grouping is True
assert supports_chunk_step_grouping(_pipeline(module))
def test_pipeline_declares_step_execution_support() -> None:
module = _load_pipeline_module()
pipeline = _pipeline(module)
@@ -2246,10 +2417,13 @@ def test_forward_rejects_a_stepwise_camera_action_script(extra_args) -> None:
pipeline(_request(sampling=sampling))
def test_stepwise_progress_metadata_and_commit_trace() -> None:
@pytest.mark.parametrize("reuse_last_step_kv", [False, True])
def test_stepwise_progress_metadata_and_commit_trace(reuse_last_step_kv: bool) -> None:
module = _load_pipeline_module()
transformer = _RecordingTransformer()
pipeline = _pipeline(module, transformer=transformer)
config = _od_config()
config.model_config["lingbot_reuse_last_step_kv"] = reuse_last_step_kv
pipeline = _pipeline(module, transformer=transformer, od_config=config)
pipeline._ar_height = 16
pipeline._ar_width = 16
state = _stepwise_state(num_frames=21)
@@ -2263,8 +2437,9 @@ def test_stepwise_progress_metadata_and_commit_trace() -> None:
assert [output.chunk_index for output in outputs] == [0, 1]
assert all(output.total_chunks == 2 for output in outputs)
assert outputs[-1].finished is True
assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True] * 2
assert [call["start_frame"] for call in transformer.calls] == [0] * 5 + [3] * 5
calls_per_chunk = 4 if reuse_last_step_kv else 5
assert [call["update_cache"] for call in transformer.calls] == ([False] * (calls_per_chunk - 1) + [True]) * 2
assert [call["start_frame"] for call in transformer.calls] == [0] * calls_per_chunk + [3] * calls_per_chunk
assert fake.commits == ["main", "main"]
for chunk_index, output in enumerate(outputs):
metadata = output.output["metadata"]["ar_diffusion"]
@@ -2325,7 +2500,10 @@ def test_stepwise_matches_tick_transformer_trace_and_latents(num_chunks) -> None
torch.testing.assert_close(step_call["camera_hidden_states"], tick_call["camera_hidden_states"])
assert stepwise_state.extra["condition"].shape[2] == 3
assert pipeline.vae.encode_inputs == []
assert [video.shape[2] for video in pipeline.vae.encoder.inputs] == ([1] + [4] * (num_chunks * 3 - 1)) * 2
# The stub encoder's history settles after its second block, so both
# modes encode two blocks and reuse the settled condition after that.
encoded_blocks = min(num_chunks, 2)
assert [video.shape[2] for video in pipeline.vae.encoder.inputs] == ([1] + [4] * (encoded_blocks * 3 - 1)) * 2
assert stepwise_fake.commits == ["main"] * num_chunks
assert len(stepwise_outputs) == num_chunks and stepwise_outputs[-1].finished
assert pipeline.vae.decode_inputs == []
@@ -2394,21 +2572,16 @@ def test_stepwise_emits_latents_without_touching_the_vae() -> None:
assert pipeline.vae.decode_inputs == []
def _capture_video_processor(monkeypatch) -> list[tuple[tuple[int, ...], str]]:
"""Record what reaches postprocess_video, and name each chunk's output."""
import diffusers.video_processor as video_processor_module
def _capture_pixel_conversion(monkeypatch) -> list[tuple[int, ...]]:
"""Record each decoded block reaching the uint8 conversion, and name its output."""
module = _load_pipeline_module()
processed: list[tuple[int, ...]] = []
processed: list[tuple[tuple[int, ...], str]] = []
def uint8_frames(video):
processed.append(tuple(video.shape))
return f"frames-{len(processed)}"
class VideoProcessor:
def __init__(self, *, vae_scale_factor):
assert vae_scale_factor == 8
def postprocess_video(self, video, *, output_type):
processed.append((tuple(video.shape), output_type))
return f"frames-{len(processed)}"
monkeypatch.setattr(video_processor_module, "VideoProcessor", VideoProcessor)
monkeypatch.setattr(module, "_uint8_frames", uint8_frames)
return processed
@@ -2422,7 +2595,7 @@ def _streaming_pipeline(module):
def test_stepwise_decodes_each_chunk_for_streaming_consumers(monkeypatch) -> None:
"""A non-latent request must stream pixels under the primary "video" key."""
module = _load_pipeline_module()
processed = _capture_video_processor(monkeypatch)
processed = _capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
state = _stepwise_state(num_frames=21)
state.sampling.output_type = "np"
@@ -2431,7 +2604,7 @@ def test_stepwise_decodes_each_chunk_for_streaming_consumers(monkeypatch) -> Non
with pipeline.bind_ar_diffusion_state(state.request_id, fake):
outputs = _run_stepwise(pipeline, state)
assert [output_type for _, output_type in processed] == ["np", "np"]
assert processed == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
assert [output.output["payload"] for output in outputs] == [{"video": "frames-1"}, {"video": "frames-2"}]
# Identity metadata survives the switch to pixel output.
assert [output.output["metadata"]["ar_diffusion"]["chunk_index"] for output in outputs] == [0, 1]
@@ -2445,7 +2618,7 @@ def test_stepwise_chunks_continue_one_sessions_temporal_decode(monkeypatch) -> N
the same latents would: 9 frames, then 12.
"""
module = _load_pipeline_module()
processed = _capture_video_processor(monkeypatch)
processed = _capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
state = _stepwise_state(num_frames=21)
state.sampling.output_type = "np"
@@ -2454,7 +2627,7 @@ def test_stepwise_chunks_continue_one_sessions_temporal_decode(monkeypatch) -> N
with pipeline.bind_ar_diffusion_state(state.request_id, fake):
_run_stepwise(pipeline, state)
assert [shape for shape, _ in processed] == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
assert processed == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
# One opening frame for the session, not one per chunk.
assert pipeline.vae.decoder.first_chunk_flags == [True] + [False] * 5
# The whole-clip decode path is not involved, and the module-owned cache
@@ -2463,10 +2636,68 @@ def test_stepwise_chunks_continue_one_sessions_temporal_decode(monkeypatch) -> N
assert pipeline.vae._feat_map == ["module-owned"]
def test_prepare_next_chunk_rejects_live_camera_with_scripted_paths() -> None:
"""Scripted/cached camera must not silently ignore mid-generation camera events."""
module = _load_pipeline_module()
pipeline = _pipeline(module, transformer=_RecordingTransformer())
pipeline._ar_height = 16
pipeline._ar_width = 16
# Scenario 1, input pre-scripted camera trajectory via action_script
state = _stepwise_state(num_frames=21)
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
pipeline.prepare_encode(state)
assert state.extra.get("camera_action_script") is not None
state.interaction_sessions["camera"] = CameraSession(has_received_input=True)
with pytest.raises(ValueError, match="camera_action_script"):
pipeline.prepare_next_chunk(state)
# Scenario 2, input pre-scripted camera trajectory via embedding cache
poses = torch.eye(4).repeat(21, 1, 1)
trajectory = _CameraTrajectory(
poses=poses,
intrinsics=torch.tensor([[100.0, 100.0, 8.0, 8.0]]).repeat(21, 1),
)
state = _stepwise_state(num_frames=21)
state.sampling.extra_args["_lingbot_camera_trajectory"] = trajectory
state.sampling.extra_args.pop("_lingbot_camera_action_script")
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)): # pyright: ignore[reportArgumentType]
pipeline.prepare_encode(state)
assert state.extra.get("camera_embedding_cache") is not None
state.interaction_sessions["camera"] = CameraSession(has_received_input=True)
with pytest.raises(ValueError, match="camera_embedding_cache"):
pipeline.prepare_next_chunk(state)
def test_peek_chunk_media_matches_streaming_decoder_frame_counts(monkeypatch) -> None:
"""Camera timelines must track the streaming decoder's 9-then-12 media counts."""
module = _load_pipeline_module()
_capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
state = _stepwise_state(num_frames=21)
state.sampling.output_type = "np"
state.sampling.fps = 16.0
fake = _FakeARState(state.request_id)
with pipeline.bind_ar_diffusion_state(state.request_id, fake):
pipeline.prepare_encode(state)
first = pipeline.peek_chunk_media(state)
assert (first.num_media_frames, first.fps, first.num_latent_frames) == (9, 16.0, 3)
pipeline.prepare_next_chunk(state)
while not state.chunk_denoise_completed:
noise = pipeline.denoise_step(None, states=[state])
pipeline.step_scheduler(state, noise)
pipeline.post_decode(state)
second = pipeline.peek_chunk_media(state)
assert (second.num_media_frames, second.fps, second.num_latent_frames) == (12, 16.0, 3)
def test_streaming_decode_state_is_owned_by_the_session(monkeypatch) -> None:
"""Decoder state is keyed by request id and released with the AR session."""
module = _load_pipeline_module()
_capture_video_processor(monkeypatch)
_capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
state = _stepwise_state(request_id="req-stream", num_frames=21)
state.sampling.output_type = "np"
@@ -2490,7 +2721,7 @@ def test_streaming_decode_state_is_owned_by_the_session(monkeypatch) -> None:
def test_streaming_decode_state_is_dropped_on_session_reset(monkeypatch) -> None:
"""A reset session must not resume the temporal context it just abandoned."""
module = _load_pipeline_module()
_capture_video_processor(monkeypatch)
_capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
state = _stepwise_state(request_id="req-reset", num_frames=21)
state.sampling.output_type = "np"
@@ -2514,7 +2745,7 @@ def test_streaming_decode_keeps_interleaved_sessions_isolated(monkeypatch) -> No
interleaving the scheduler actually produces.
"""
module = _load_pipeline_module()
processed = _capture_video_processor(monkeypatch)
processed = _capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
states = []
for request_id in ("req-a", "req-b"):
@@ -2547,7 +2778,7 @@ def test_streaming_decode_keeps_interleaved_sessions_isolated(monkeypatch) -> No
assert pipeline.vae.decoder.first_chunk_flags == (
[True, False, False] + [True, False, False] + [False] * 3 + [False] * 3
)
assert [shape for shape, _ in processed] == [
assert processed == [
(1, 3, 9, 16, 16),
(1, 3, 9, 16, 16),
(1, 3, 12, 16, 16),
@@ -2558,7 +2789,7 @@ def test_streaming_decode_keeps_interleaved_sessions_isolated(monkeypatch) -> No
def test_streaming_decode_failure_drops_the_half_advanced_cache(monkeypatch) -> None:
"""A failed chunk must not leave a cache a later chunk would continue from."""
module = _load_pipeline_module()
_capture_video_processor(monkeypatch)
_capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
state = _stepwise_state(request_id="req-boom", num_frames=21)
state.sampling.output_type = "np"
@@ -2592,13 +2823,14 @@ def test_streaming_decode_receives_rescaled_latents(monkeypatch) -> None:
never reaches, so this asserts on what ``post_quant_conv`` received.
"""
module = _load_pipeline_module()
_capture_video_processor(monkeypatch)
_capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
state = _stepwise_state(num_frames=21)
state.sampling.output_type = "np"
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
pipeline.prepare_encode(state)
pipeline.prepare_next_chunk(state)
while not state.chunk_denoise_completed:
pipeline.step_scheduler(state, pipeline.denoise_step(None, states=[state]))
model_space = state.latents.clone()
@@ -2624,7 +2856,7 @@ def test_streaming_decode_is_reported_to_the_profiler(monkeypatch) -> None:
perf regression in the streaming decoder would be too.
"""
module = _load_pipeline_module()
_capture_video_processor(monkeypatch)
_capture_pixel_conversion(monkeypatch)
pipeline = _pipeline(
module,
transformer=_RecordingTransformer(),
@@ -2652,21 +2884,18 @@ def test_streaming_decode_is_reported_to_the_profiler(monkeypatch) -> None:
def test_streaming_decode_falls_back_when_the_vae_would_tile(monkeypatch) -> None:
"""Tiled decode drives its own cache, so a tiling shape keeps per-chunk decode."""
module = _load_pipeline_module()
processed = _capture_video_processor(monkeypatch)
processed = _capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
# A latent of 2x2 exceeds a one-latent-cell tile, so vae.decode would tile.
_enable_vae_tiling(pipeline, tile_sample_min=8)
outputs = [
pipeline._decode_chunk_to_pixels(torch.zeros(1, 16, 3, 2, 2), output_type="np", session_id="tiled")
for _ in range(2)
]
outputs = [pipeline._decode_chunk_to_pixels(torch.zeros(1, 16, 3, 2, 2), session_id="tiled") for _ in range(2)]
# One whole-clip decode per AR block, each seeing only that block's frames.
assert [tuple(latents.shape) for latents in pipeline.vae.decode_inputs] == [(1, 16, 3, 2, 2)] * 2
assert pipeline.vae.decoder.first_chunk_flags == []
assert pipeline._streaming_decode_states == {}
# Nine frames per chunk instead of 9 then 12: the frame loss being removed.
assert [shape for shape, _ in processed] == [(1, 3, 9, 16, 16), (1, 3, 9, 16, 16)]
assert processed == [(1, 3, 9, 16, 16), (1, 3, 9, 16, 16)]
assert outputs == ["frames-1", "frames-2"]
@@ -2678,7 +2907,7 @@ def test_streaming_decode_survives_tiling_enabled_below_the_tile_threshold(monke
on a configuration that would never have tiled.
"""
module = _load_pipeline_module()
processed = _capture_video_processor(monkeypatch)
processed = _capture_pixel_conversion(monkeypatch)
pipeline = _streaming_pipeline(module)
# A 2x2 latent is within a tile this size, so vae.decode would not tile.
_enable_vae_tiling(pipeline, tile_sample_min=256)
@@ -2690,7 +2919,7 @@ def test_streaming_decode_survives_tiling_enabled_below_the_tile_threshold(monke
assert pipeline.vae.decode_inputs == []
assert pipeline.vae.decoder.first_chunk_flags == [True] + [False] * 5
assert [shape for shape, _ in processed] == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
assert processed == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
def test_registry_and_model_exports_resolve_official_pipeline_class_name() -> None:
@@ -2833,3 +3062,193 @@ def test_stepwise_requests_with_the_same_prompt_share_one_encode() -> None:
assert encoded == ["move through the room"]
assert states[0].prompt_embeds is states[1].prompt_embeds
def test_session_text_caches_publish_key_only_after_a_successful_build() -> None:
module = _load_pipeline_module()
pipeline = _pipeline(module)
state = module._LingBotARSessionState()
invalidations: list[bool] = []
fail_next = {"raise": False}
def fake_text_caches(prompt_embeds, *, invalidate):
invalidations.append(invalidate)
if fail_next["raise"]:
fail_next["raise"] = False
raise RuntimeError("injected failure inside _ar_text_caches")
return ["caches"]
pipeline._ar_text_caches = fake_text_caches
embeds = torch.ones(1, 512, 8)
caches = lambda prompt, length=512, e=embeds: pipeline._session_text_caches( # noqa: E731
e, prompt=prompt, max_sequence_length=length, session_state=state
)
# First build of a session: runner state is empty -> no invalidation, key published.
caches("A")
assert invalidations == [False] and state.text_cache_key == ("A", 512, str(embeds.dtype))
assert state.text_cache_dirty is False
# Same inputs: reuse without invalidation.
caches("A")
assert invalidations == [False, False]
# Regression (Codex review): B's build fails after invalidating -> no key is
# left behind, so retrying A must invalidate again instead of consuming
# partial/mixed K/V.
fail_next["raise"] = True
with pytest.raises(RuntimeError, match="injected failure"):
caches("B")
assert state.text_cache_key is None and state.text_cache_dirty is True
caches("A")
assert invalidations[-2:] == [True, True] and state.text_cache_key == ("A", 512, str(embeds.dtype))
assert state.text_cache_dirty is False
# The key covers every embedding-defining input: same text, other length or dtype -> rebuild.
caches("A", length=256)
caches("A", e=embeds.to(torch.bfloat16))
assert invalidations[-2:] == [True, True]
# ---------------------------------------------------------------------------
# a source image is decoded once while the file behind it is unchanged
# ---------------------------------------------------------------------------
def _write_png(path, colour=(10, 20, 30)):
Image.new("RGB", (64, 64), colour).save(path)
return path
def _counting_decode(module):
"""Replace the file decode with a counter, so the tests measure decodes rather than pixels."""
decoded: list[str] = []
original = module._decode_source_image_file
def decode(path):
decoded.append(os.fspath(path))
return Image.new("RGB", (64, 64), (1, 2, 3))
module._decode_source_image_file = decode
return decoded, original
def test_the_same_source_image_is_decoded_once(tmp_path: Path) -> None:
module = _load_pipeline_module()
module._decode_source_image_fingerprinted.cache_clear()
path = _write_png(tmp_path / "frame.png")
decoded, original = _counting_decode(module)
try:
first = module._load_source_image(path)
second = module._load_source_image(path)
assert decoded == [os.fspath(os.path.realpath(path))] or len(decoded) == 1
# Every caller owns its copy: mutating one must not reach the next caller or the cache.
assert first is not second
first.putpixel((0, 0), (255, 255, 255))
assert module._load_source_image(path).getpixel((0, 0)) == (1, 2, 3)
assert len(decoded) == 1
finally:
module._decode_source_image_file = original
module._decode_source_image_fingerprinted.cache_clear()
def test_a_rewritten_source_image_is_decoded_again(tmp_path: Path) -> None:
module = _load_pipeline_module()
module._decode_source_image_fingerprinted.cache_clear()
path = _write_png(tmp_path / "frame.png")
decoded, original = _counting_decode(module)
try:
module._load_source_image(path)
module._load_source_image(path)
assert len(decoded) == 1
# Same path, different bytes: the mtime/size in the key must force a fresh decode.
os.utime(path, ns=(0, 0))
_write_png(path, colour=(200, 100, 50))
module._load_source_image(path)
assert len(decoded) == 2
finally:
module._decode_source_image_file = original
module._decode_source_image_fingerprinted.cache_clear()
def test_an_image_rewritten_during_the_decode_is_never_cached(tmp_path: Path) -> None:
module = _load_pipeline_module()
module._decode_source_image_fingerprinted.cache_clear()
path = _write_png(tmp_path / "frame.png")
decoded: list[str] = []
original = module._decode_source_image_file
def decode_then_rewrite(p):
decoded.append(os.fspath(p))
_write_png(path, colour=(len(decoded), 0, 0))
os.utime(path, ns=(len(decoded) * 1_000_000, len(decoded) * 1_000_000))
return Image.new("RGB", (64, 64), (1, 2, 3))
module._decode_source_image_file = decode_then_rewrite
try:
module._load_source_image(path)
# The entry is keyed by the fingerprint taken before the decode, which the rewrite invalidated: the
# next call must decode again rather than serve the image that was decoded mid-rewrite.
module._load_source_image(path)
assert len(decoded) == 2
finally:
module._decode_source_image_file = original
module._decode_source_image_fingerprinted.cache_clear()
def test_an_unstattable_path_keeps_upstream_validation(tmp_path: Path) -> None:
module = _load_pipeline_module()
module._decode_source_image_fingerprinted.cache_clear()
missing = tmp_path / "does-not-exist.png"
# The uncached path owns the error contract; caching must not change which exception a caller sees.
with pytest.raises(ValueError):
module._load_source_image(missing)
assert module._decode_source_image_fingerprinted.cache_info().currsize == 0
def test_the_text_cache_is_sized_by_cross_attention_heads_not_the_self_attention_share() -> None:
"""Cross-attention replicates heads per rank, so its pool must not inherit the self-attention head count."""
module = _load_pipeline_module()
pipeline = _pipeline(module)
pipeline.transformer.blocks[0].self_attn = SimpleNamespace(num_sp_heads=1)
pipeline.transformer.blocks[0].cross_attn = SimpleNamespace(num_local_heads=4)
spec = pipeline.ar_diffusion_kv_cache_spec()
assert spec.num_kv_heads == 1
assert spec.cross_attention_kv_heads == {"text": 4}
assert spec.cross_attention_lengths == {"text": 512}
@pytest.mark.parametrize("output_type", ["np", "pil"])
def test_streamed_chunk_is_uint8_frames_whatever_pixel_output_type_was_asked(monkeypatch, output_type) -> None:
"""A realtime tick delivers (F, H, W, 3) uint8 for every pixel output type; only latent differs."""
module = _load_pipeline_module()
pipeline = _streaming_pipeline(module)
state = _stepwise_state(num_frames=21)
state.sampling.output_type = output_type
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
outputs = _run_stepwise(pipeline, state)
videos = [output.output["payload"]["video"] for output in outputs]
assert [type(video) for video in videos] == [np.ndarray, np.ndarray]
assert [video.dtype for video in videos] == [np.uint8, np.uint8]
assert [video.shape for video in videos] == [(9, 16, 16, 3), (12, 16, 16, 3)]
def test_uint8_frames_match_the_pil_conversion_byte_for_byte() -> None:
"""The device-side conversion is the PIL path's arithmetic, so the bytes are the same."""
from diffusers.video_processor import VideoProcessor
module = _load_pipeline_module()
generator = torch.Generator().manual_seed(7)
# Cover the clamp on both sides and values that sit exactly on rounding edges.
video = torch.rand(1, 3, 5, 8, 8, generator=generator) * 2.4 - 1.2
video[0, :, 0, 0, :4] = torch.tensor([-1.0, 1.0, 0.0, 1 / 255 - 1.0])
reference = VideoProcessor(vae_scale_factor=8).postprocess_video(video, output_type="pil")[0]
expected = np.stack([np.asarray(frame) for frame in reference])
frames = module._uint8_frames(video)
assert frames.dtype == np.uint8 and frames.shape == (5, 8, 8, 3)
assert frames.flags["C_CONTIGUOUS"]
np.testing.assert_array_equal(frames, expected)
@@ -0,0 +1,239 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from contextlib import nullcontext
from types import SimpleNamespace
import pytest
import torch
from diffusers.models.embeddings import apply_rotary_emb
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@pytest.fixture(autouse=True)
def _clear_qk_rope_signature_caches():
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
longcat._FAILED_QK_ROPE_SIGNATURES.clear()
yield
longcat._FAILED_QK_ROPE_SIGNATURES.clear()
def _inputs(sequence: int = 5, head_dim: int = 8):
q = torch.randn(2, sequence, 3, head_dim, dtype=torch.bfloat16)
k = torch.randn_like(q)
cos = torch.randn(sequence, head_dim)
sin = torch.randn_like(cos)
return q, k, (cos, sin)
def _reference(q, k, rotary_emb):
return (
apply_rotary_emb(q, rotary_emb, sequence_dim=1),
apply_rotary_emb(k, rotary_emb, sequence_dim=1),
)
def test_prepare_rotary_emb_resolves_threshold_once_only_when_enabled(monkeypatch):
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
calls = 0
def resolve(default):
nonlocal calls
calls += 1
assert default == 512
return 37
monkeypatch.setattr(longcat, "fused_qk_norm_rope_min_tokens", resolve)
txt_cos, txt_sin = torch.randn(2, 8), torch.randn(2, 8)
img_cos, img_sin = torch.randn(3, 8), torch.randn(3, 8)
native = longcat._prepare_rotary_emb(txt_cos, txt_sin, img_cos, img_sin, enable_fusion=False)
fused = longcat._prepare_rotary_emb(txt_cos, txt_sin, img_cos, img_sin, enable_fusion=True)
assert calls == 1
assert len(native) == 2
assert len(fused) == 3 and fused[2] == 37
assert torch.equal(native[0], fused[0])
assert torch.equal(native[1], fused[1])
@pytest.mark.parametrize(
("mode", "sp_size"),
[
("unsupported", 1),
("compile", 1),
("grad", 1),
("capture", 1),
("sequence_parallel", 2),
("below_threshold", 1),
],
)
def test_qk_rope_non_eager_modes_use_exact_original_path(monkeypatch, mode, sp_size):
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
torch.manual_seed(1)
q, k, rotary_emb = _inputs()
expected = _reference(q, k, rotary_emb)
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: mode == "compile")
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: mode == "capture")
monkeypatch.setattr(longcat, "fused_qk_rope_supported", lambda *args: mode != "unsupported")
monkeypatch.setattr(
longcat,
"fused_qk_rope",
lambda *args: (_ for _ in ()).throw(AssertionError("fallback called fused op")),
)
context = nullcontext() if mode == "grad" else torch.no_grad()
min_tokens = q.shape[0] * q.shape[1] + 1 if mode == "below_threshold" else 0
with context:
actual = longcat._apply_qk_rope(q, k, rotary_emb, sp_size, min_tokens)
assert torch.equal(actual[0], expected[0])
assert torch.equal(actual[1], expected[1])
def test_qk_rope_eligible_path_skips_runtime_parity_check(monkeypatch):
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
torch.manual_seed(2)
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: False)
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
monkeypatch.setattr(longcat, "fused_qk_rope_supported", lambda *args: True)
fused_calls = 0
def fused(query, key, cos, sin):
nonlocal fused_calls
fused_calls += 1
return query + 1, key + 2
monkeypatch.setattr(
longcat,
"_apply_qk_rope_reference",
lambda *args: (_ for _ in ()).throw(AssertionError("eligible path ran the native parity check")),
)
monkeypatch.setattr(longcat, "fused_qk_rope", fused)
with torch.no_grad():
q, k, rotary_emb = _inputs()
actual = longcat._apply_qk_rope(q, k, rotary_emb, 1, 0)
assert fused_calls == 1
assert torch.equal(actual[0], q + 1)
assert torch.equal(actual[1], k + 2)
def test_qk_rope_launch_failure_permanently_falls_back(monkeypatch):
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
torch.manual_seed(3)
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: False)
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
monkeypatch.setattr(longcat, "fused_qk_rope_supported", lambda *args: True)
fused_calls = 0
def broken_fused(query, key, cos, sin):
nonlocal fused_calls
fused_calls += 1
raise RuntimeError("kernel failed")
monkeypatch.setattr(longcat, "fused_qk_rope", broken_fused)
q, k, rotary_emb = _inputs()
expected = _reference(q, k, rotary_emb)
with torch.no_grad():
first = longcat._apply_qk_rope(q, k, rotary_emb, 1, 0)
second = longcat._apply_qk_rope(q, k, rotary_emb, 1, 0)
assert fused_calls == 1
assert len(longcat._FAILED_QK_ROPE_SIGNATURES) == 1
for actual in (first, second):
assert torch.equal(actual[0], expected[0])
assert torch.equal(actual[1], expected[1])
def test_failed_qk_rope_signature_cache_is_bounded():
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
for index in range(longcat._FAILED_QK_ROPE_SIGNATURES_MAX_SIZE + 1):
longcat._record_failed_qk_rope_signature((index,))
assert len(longcat._FAILED_QK_ROPE_SIGNATURES) == longcat._FAILED_QK_ROPE_SIGNATURES_MAX_SIZE
assert (0,) not in longcat._FAILED_QK_ROPE_SIGNATURES
assert (longcat._FAILED_QK_ROPE_SIGNATURES_MAX_SIZE,) in longcat._FAILED_QK_ROPE_SIGNATURES
class _FakeQKV(torch.nn.Module):
def __init__(self, heads: int, head_dim: int) -> None:
super().__init__()
self.num_heads = heads
self.num_kv_heads = heads
self.width = heads * head_dim
def forward(self, x):
return torch.cat((x, x + 1, x + 2), dim=-1), None
class _AddNorm(torch.nn.Module):
def __init__(self, value: float) -> None:
super().__init__()
self.value = value
def forward(self, x):
return x + self.value
class _CaptureAttention(torch.nn.Module):
def forward(self, query, key, value, metadata=None):
del key, value, metadata
return query
class _PassthroughLinear(torch.nn.Module):
def forward(self, x):
return x, None
def test_dual_attention_keeps_native_norm_and_text_image_concat_order(monkeypatch):
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
attention = longcat.LongCatImageAttention.__new__(longcat.LongCatImageAttention)
torch.nn.Module.__init__(attention)
attention.parallel_config = SimpleNamespace(sequence_parallel_size=1)
attention.head_dim = 4
attention.added_kv_proj_dim = 8
attention.to_qkv = _FakeQKV(2, 4)
attention.add_kv_proj = _FakeQKV(2, 4)
attention.norm_q = _AddNorm(10)
attention.norm_k = _AddNorm(20)
attention.norm_added_q = _AddNorm(100)
attention.norm_added_k = _AddNorm(200)
attention.attn = _CaptureAttention()
attention.to_out = _PassthroughLinear()
attention.to_add_out = _PassthroughLinear()
captured = []
def paired(query, key, rotary_emb, sp_size, min_tokens):
captured.append((query.clone(), key.clone(), rotary_emb, sp_size, min_tokens))
return query, key
monkeypatch.setattr(longcat, "_apply_qk_rope", paired)
image = torch.randn(1, 3, 8)
text = torch.randn(1, 2, 8)
rotary_pair = (torch.randn(5, 4), torch.randn(5, 4))
rotary_emb = (*rotary_pair, 0)
image_out, text_out = attention(image, text, rotary_emb)
image_q = image.unflatten(-1, (2, 4))
text_q = text.unflatten(-1, (2, 4))
expected_q = torch.cat((text_q + 100, image_q + 10), dim=1)
expected_k = torch.cat((text_q + 1 + 200, image_q + 1 + 20), dim=1)
assert len(captured) == 1
assert torch.equal(captured[0][0], expected_q)
assert torch.equal(captured[0][1], expected_k)
assert captured[0][2][0] is rotary_pair[0]
assert captured[0][2][1] is rotary_pair[1]
assert captured[0][3:] == (1, 0)
assert image_out.shape == image.shape
assert text_out.shape == text.shape
@@ -14,12 +14,13 @@ from tests.diffusion.models.magi2.test_native_packing import (
)
from tests.helpers.mark import hardware_marks
from vllm_omni.diffusion.models.magi2.sampler_magi2 import CFGConfig
from vllm_omni.platforms import current_omni_platform
pytestmark = [
pytest.mark.diffusion,
pytest.mark.core_model,
*hardware_marks(res={"cuda": "L4"}, num_cards=1),
pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA"),
pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="requires CUDA"),
]
@@ -36,6 +36,7 @@ from vllm_omni.diffusion.offloader.distributed_layerwise_backend import (
DistributedLayerwiseOffloadBackend,
DistributedLayerwiseOffloadHook,
)
from vllm_omni.diffusion.offloader.plan_resolver import ResolvedComponent
@dataclass(frozen=True)
@@ -46,12 +47,6 @@ class _WeightSourceStub:
prefix: str
@dataclass(frozen=True)
class _PipelineModulesStub:
dits: list[nn.Module]
dit_names: list[str]
_WORLD_SIZE = 4
pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model]
@@ -189,7 +184,7 @@ def _load_mmap_transform_and_reconstruct(
backend._mmap_transforms_by_tensor_id = {}
backend._load_weights_via_mmap(
pipeline,
_PipelineModulesStub(dits=[target], dit_names=["transformer"]),
(ResolvedComponent(path="transformer", module=target, selected=True),),
plan_result.plan,
)
@@ -90,6 +90,21 @@ def test_hsdp_policy_shards_individual_preview_layers():
assert not condition("pre_adapter", model.pre_adapter)
def test_layerwise_offload_contract() -> None:
from vllm_omni.diffusion.offloader import get_blocks_from_dit
model = Magi2PreviewTransformer(_tiny_config(num_layers=2))
assert model._layerwise_offload_blocks_attrs == ["block"]
assert "block" in dict(model.named_children())
assert list(model.block) == list(model.block.layers)
assert model.layers is model.block.layers
attr_names, blocks = get_blocks_from_dit(model)
assert attr_names == ["block"]
assert blocks == list(model.block.layers)
def test_data_proxy_keeps_output_layout_request_scoped() -> None:
proxy = Magi2DataProxy(Magi2PreviewDataProxyConfig(time_channel_dim=0))
@@ -0,0 +1,392 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from dataclasses import dataclass
import pytest
import torch
from torch import nn
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig, TransformerConfig
from vllm_omni.diffusion.models.mammoth_moda2 import pipeline_mammothmoda2_dit
from vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit import (
MammothModa2DiTPipeline,
_build_mammoth_config,
_root_weight_source,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _raw_config() -> dict:
return {
"model_type": "mammothmoda2",
"llm_config": {
"model_type": "mammothmoda2_qwen2_5_vl",
"text_config": {
"model_type": "mammothmoda2_qwen2_5_vl_text",
"hidden_size": 8,
"gen_vocab_start_index": 100,
},
},
"gen_vae_config": {"block_out_channels": [8, 8]},
"gen_dit_config": {"hidden_size": 8, "in_channels": 4},
}
def _od_config() -> OmniDiffusionConfig:
return OmniDiffusionConfig(
model="/models/MammothModa2-Preview",
model_class_name="MammothModa2DiTPipeline",
tf_model_config=TransformerConfig.from_dict(_raw_config()),
)
def test_build_mammoth_config_uses_shared_transformer_projection() -> None:
config = _build_mammoth_config(_od_config())
assert config.model_type == "mammothmoda2"
assert config.get_text_config().hidden_size == 8
assert config.gen_dit_config == {"hidden_size": 8, "in_channels": 4}
def test_build_mammoth_config_rejects_empty_shared_projection() -> None:
config = _od_config()
config.tf_model_config = TransformerConfig()
with pytest.raises(ValueError, match="root checkpoint config"):
_build_mammoth_config(config)
def test_root_weight_source_loads_combined_checkpoint_once() -> None:
source = _root_weight_source(_od_config())
assert source.model_or_path == "/models/MammothModa2-Preview"
assert source.subfolder is None
assert source.prefix == ""
assert source.fall_back_to_pt is True
def test_root_weight_source_forwards_revision() -> None:
config = _od_config()
config.revision = "rev-7"
assert _root_weight_source(config).revision == "rev-7"
def test_pipeline_declares_native_components_and_single_request_mode_only() -> None:
assert MammothModa2DiTPipeline._dit_modules == ["gen_transformer"]
assert MammothModa2DiTPipeline._encoder_modules == ["gen_image_condition_refiner"]
assert MammothModa2DiTPipeline._vae_modules == ["gen_vae"]
assert MammothModa2DiTPipeline.supports_request_batch is False
assert MammothModa2DiTPipeline.supports_step_execution is False
def test_mammoth_postprocess_denormalizes_nonnegative_raw_vae_output() -> None:
factory = getattr(pipeline_mammothmoda2_dit, "get_mammoth_moda2_post_process_func", None)
assert factory is not None
images = factory(_od_config())(torch.zeros(1, 3, 2, 2))
assert len(images) == 1
assert images[0].getpixel((0, 0)) == (128, 128, 128)
def test_mammoth_postprocess_is_registered() -> None:
from vllm_omni.diffusion.registry import _DIFFUSION_POST_PROCESS_FUNCS
assert _DIFFUSION_POST_PROCESS_FUNCS["MammothModa2DiTPipeline"] == "get_mammoth_moda2_post_process_func"
def test_root_weight_source_rejects_missing_model_path() -> None:
config = _od_config()
config.model = None
with pytest.raises(ValueError, match="model path"):
_root_weight_source(config)
def _pipeline_shell() -> MammothModa2DiTPipeline:
pipeline = object.__new__(MammothModa2DiTPipeline)
nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.config = _build_mammoth_config(_od_config())
pipeline._llm_hidden_size = 8
return pipeline
def _batch(
*,
request_id: str = "req-7",
prompt: object | None = None,
sampling: OmniDiffusionSamplingParams | None = None,
) -> DiffusionRequestBatch:
if prompt is None:
prompt = {
"prompt": "",
"height": 32,
"width": 48,
"additional_information": {
"full_hidden_states": torch.arange(32, dtype=torch.float32).reshape(4, 8),
"full_token_ids": [10, 11, 100, 101],
"answer_start_index": 2,
},
}
if sampling is None:
sampling = OmniDiffusionSamplingParams(
height=32,
width=48,
seed=42,
guidance_scale=4.0,
num_inference_steps=7,
extra_args={"cfg_range": [0.2, 0.8]},
)
return DiffusionRequestBatch([OmniDiffusionRequest(prompt=prompt, sampling_params=sampling, request_id=request_id)])
def test_parse_request_uses_standard_sampling_fields() -> None:
parsed = _pipeline_shell()._parse_request(_batch())
assert parsed.request_id == "req-7"
assert (parsed.height, parsed.width) == (32, 48)
assert parsed.num_inference_steps == 7
assert parsed.text_guidance_scale == 4.0
assert parsed.cfg_range == (0.2, 0.8)
assert parsed.seed == 42
assert parsed.answer_start_index == 2
def test_parse_request_prefers_legacy_sampling_overrides() -> None:
sampling = OmniDiffusionSamplingParams(
guidance_scale=3.0,
num_inference_steps=5,
seed=9,
extra_args={"text_guidance_scale": 6.0, "num_inference_steps": 11, "cfg_range": [0.0, 0.5]},
)
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
assert parsed.text_guidance_scale == 6.0
assert parsed.num_inference_steps == 11
assert parsed.cfg_range == (0.0, 0.5)
def test_parse_request_falls_back_to_request_level_sampling_values() -> None:
prompt = _batch().prompts[0]
prompt["additional_information"].update(
text_guidance_scale=[1.5],
num_inference_steps=[3],
cfg_range=[0.25, 0.75],
)
parsed = _pipeline_shell()._parse_request(_batch(prompt=prompt, sampling=OmniDiffusionSamplingParams()))
assert parsed.text_guidance_scale == 1.5
assert parsed.num_inference_steps == 3
assert parsed.cfg_range == (0.25, 0.75)
def test_parse_request_standard_fields_precede_request_level_fallbacks() -> None:
prompt = _batch().prompts[0]
prompt["additional_information"].update(
text_guidance_scale=[1.5],
num_inference_steps=[3],
)
sampling = OmniDiffusionSamplingParams(guidance_scale=4.0, num_inference_steps=7)
parsed = _pipeline_shell()._parse_request(_batch(prompt=prompt, sampling=sampling))
assert parsed.text_guidance_scale == 4.0
assert parsed.num_inference_steps == 7
def test_parse_request_rejects_multiple_requests_and_outputs() -> None:
pipeline = _pipeline_shell()
batch = _batch()
batch.requests.append(_batch(request_id="req-8").requests[0])
with pytest.raises(ValueError, match="exactly one request"):
pipeline._parse_request(batch)
with pytest.raises(ValueError, match="num_outputs_per_prompt=1"):
pipeline._parse_request(_batch(sampling=OmniDiffusionSamplingParams(num_outputs_per_prompt=2)))
@pytest.mark.parametrize(
("cfg_range", "message"),
[
([0.5], "two values"),
([-0.1, 0.5], "0 <= start <= end <= 1"),
([0.7, 0.2], "0 <= start <= end <= 1"),
([0.2, 1.1], "0 <= start <= end <= 1"),
],
)
def test_parse_request_rejects_invalid_cfg_range(cfg_range, message) -> None:
batch = _batch(sampling=OmniDiffusionSamplingParams(extra_args={"cfg_range": cfg_range}))
with pytest.raises(ValueError, match=message):
_pipeline_shell()._parse_request(batch)
def test_parse_request_requires_real_ar_conditions() -> None:
with pytest.raises(ValueError, match="req-missing"):
_pipeline_shell()._parse_request(_batch(request_id="req-missing", prompt={"prompt": "draw a cat"}))
def test_parse_request_rejects_hidden_state_token_count_mismatch() -> None:
batch = _batch()
batch.prompts[0]["additional_information"]["full_hidden_states"] = torch.zeros(3, 8)
with pytest.raises(ValueError, match="hidden-state/token-count mismatch"):
_pipeline_shell()._parse_request(batch)
@pytest.mark.parametrize(
("height", "width", "message"),
[
(0, 32, "Invalid image size"),
(32, -1, "Invalid image size"),
(30, 32, "multiples of 16"),
(32, 31, "multiples of 16"),
],
)
def test_parse_request_rejects_invalid_dimensions(height, width, message) -> None:
batch = _batch()
batch.prompts[0].update(height=height, width=width)
with pytest.raises(ValueError, match=message):
_pipeline_shell()._parse_request(batch)
def test_parse_request_rejects_explicit_zero_steps() -> None:
with pytest.raises(ValueError, match="num_inference_steps must be positive"):
_pipeline_shell()._parse_request(_batch(sampling=OmniDiffusionSamplingParams(num_inference_steps=0)))
@pytest.mark.parametrize(("standard_steps", "expected_steps"), [(7, 7), (None, 50)])
def test_parse_request_falls_through_null_legacy_step_count(standard_steps, expected_steps) -> None:
sampling = OmniDiffusionSamplingParams(num_inference_steps=standard_steps, extra_args={"num_inference_steps": None})
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
assert parsed.num_inference_steps == expected_steps
@pytest.mark.parametrize(("standard_guidance", "expected_guidance"), [(4.0, 4.0), (None, 9.0)])
def test_parse_request_falls_through_null_legacy_guidance(standard_guidance, expected_guidance) -> None:
sampling = OmniDiffusionSamplingParams(guidance_scale=standard_guidance, extra_args={"text_guidance_scale": None})
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
assert parsed.text_guidance_scale == expected_guidance
def test_parse_request_defaults_null_cfg_range() -> None:
sampling = OmniDiffusionSamplingParams(extra_args={"cfg_range": None})
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
assert parsed.cfg_range == (0.0, 1.0)
@pytest.mark.parametrize("generators", [[], [torch.Generator(), torch.Generator()]])
def test_parse_request_rejects_invalid_generator_list_cardinality(generators) -> None:
sampling = OmniDiffusionSamplingParams(generator=generators)
with pytest.raises(ValueError, match="exactly one generator.*req-generators"):
_pipeline_shell()._parse_request(_batch(request_id="req-generators", sampling=sampling))
def test_parse_request_synthesizes_dummy_ar_conditions() -> None:
batch = _batch(
request_id="dummy_req_id",
prompt={"prompt": "dummy run"},
sampling=OmniDiffusionSamplingParams(height=512, width=512, seed=1, guidance_scale=0.0, num_inference_steps=2),
)
parsed = _pipeline_shell()._parse_request(batch)
assert parsed.full_hidden_states.shape == (2, 8)
assert parsed.full_token_ids == [0, 100]
assert parsed.answer_start_index == 1
@dataclass
class _FakeTransformerConfig:
in_channels: int = 4
@dataclass
class _FakeTimeCaptionEmbed:
image_embedder: nn.Module | None = None
class _FakeTransformer(nn.Module):
def __init__(self) -> None:
super().__init__()
self.anchor = nn.Parameter(torch.zeros(1))
self.config = _FakeTransformerConfig()
self.time_caption_embed = _FakeTimeCaptionEmbed()
self.calls = 0
def forward(self, *, hidden_states, **kwargs):
self.calls += 1
return torch.zeros_like(hidden_states)
@dataclass
class _FakeVaeConfig:
scaling_factor: float | None = None
shift_factor: float | None = None
class _FakeVae(nn.Module):
def __init__(self) -> None:
super().__init__()
self.anchor = nn.Parameter(torch.zeros(1))
self.config = _FakeVaeConfig()
def decode(self, latents, return_dict=False):
assert return_dict is False
return (torch.zeros(1, 3, 32, 48, dtype=latents.dtype),)
class _FakeScheduler:
def __init__(self) -> None:
self.timesteps = torch.tensor([])
self.requested_steps = None
def set_timesteps(self, *, num_inference_steps, device, num_tokens):
self.requested_steps = num_inference_steps
self.timesteps = torch.arange(num_inference_steps, device=device, dtype=torch.float32)
def step(self, model_pred, timestep, latents, return_dict=False):
assert return_dict is False
return (latents - model_pred,)
def test_forward_returns_diffusion_output_with_request_sampling(mocker) -> None:
pipeline = _pipeline_shell()
pipeline.gen_transformer = _FakeTransformer()
pipeline.gen_image_condition_refiner = None
pipeline.gen_vae = _FakeVae()
pipeline.gen_freqs_cis = torch.zeros(1)
scheduler = _FakeScheduler()
captured = {}
def fake_randn_tensor(shape, *, generator, device, dtype):
captured["shape"] = shape
captured["seed"] = generator.initial_seed()
return torch.zeros(shape, device=device, dtype=dtype)
module = "vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit"
mocker.patch(f"{module}.FlowMatchEulerDiscreteScheduler", return_value=scheduler)
mocker.patch(f"{module}.randn_tensor", side_effect=fake_randn_tensor)
result = pipeline.forward(
_batch(
sampling=OmniDiffusionSamplingParams(
height=32, width=48, seed=42, guidance_scale=1.0, num_inference_steps=2
)
)
)
assert isinstance(result, DiffusionOutput)
assert result.output.shape == (1, 3, 32, 48)
assert captured["shape"] == (1, 4, 4, 6)
assert captured["seed"] == 42
assert scheduler.requested_steps == 2
assert pipeline.gen_transformer.calls == 2
def test_forward_rejects_missing_visual_tokens_before_model_access() -> None:
prompt = {
"prompt": "",
"additional_information": {
"full_hidden_states": torch.zeros(3, 8),
"full_token_ids": [10, 11, 12],
"answer_start_index": 2,
},
}
with pytest.raises(ValueError, match="no visual-token hidden states.*req-empty"):
_pipeline_shell().forward(_batch(request_id="req-empty", prompt=prompt))
@@ -0,0 +1,679 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import json
from collections.abc import Callable, Iterable
import pytest
import torch
import torch.nn as nn
from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, DiffusionParallelConfig, OmniDiffusionConfig
from vllm_omni.diffusion.models.minimax_h3.fasth3_checkpoint import (
FASTH3_BASE_MODEL,
FASTH3_V2_BASE_SCHEDULE,
FASTH3_V2_MODEL_ID,
FastH3CheckpointSpec,
)
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
MiniMaxH3DiTArchConfig,
MiniMaxH3DiTModel,
)
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
resolve_minimax_h3_diffusion_model_path,
)
from vllm_omni.diffusion.sched.sigma_schedule import DMD2SigmaSchedule
from vllm_omni.errors import OmniClientError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.lora.request import LoRARequest
from vllm_omni.model_executor.models.minimax_h3.checkpoint import (
is_minimax_h3_modular,
resolve_minimax_h3_model_root,
)
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
def _fastvideo_metadata() -> dict[str, object]:
"""Return the release-owned metadata read from fastvideo_inference.json."""
return {
"model_id": FASTH3_V2_MODEL_ID,
"schema_version": "fasth3-inference-contract-v1",
"dmd_denoising_steps": [999, 874, 749, 624, 500, 375, 250, 125],
"transformer_forwards": 8,
"video_scheduler_shift": 10.0,
"audio_scheduler_shift": 3.0,
"guidance_scale": 1.0,
"attention_backend": "VIDEO_SPARSE_ATTN_H3",
"vsa_sparsity": 0.8,
"vsa_tile_size": 64,
"task": "t2av",
}
def _sampling(
*,
num_inference_steps: int | str | None = 8,
guidance_scale: float | None = 1.0,
extra_args: dict[str, object] | None = None,
lora_request: LoRARequest | None = None,
timesteps: torch.Tensor | None = None,
sigmas: list[float] | None = None,
) -> OmniDiffusionSamplingParams:
return OmniDiffusionSamplingParams(
# Keep the string form to exercise malformed client input validation.
num_inference_steps=num_inference_steps, # type: ignore[arg-type]
guidance_scale=guidance_scale,
extra_args=extra_args or {},
lora_request=lora_request,
timesteps=timesteps,
sigmas=sigmas,
)
def _od_config(
*,
backend: str = "FASTVIDEO_VSA",
topk: int | None = None,
per_role: dict[str, AttentionSpec] | None = None,
lora_path: str | None = None,
ring_degree: int = 1,
allgather_degree: int = 1,
) -> OmniDiffusionConfig:
attention_config = AttentionConfig(
default=AttentionSpec(backend=backend, fastvideo_vsa_topk=topk),
per_role=per_role or {},
)
return OmniDiffusionConfig(
diffusion_attention_config=attention_config,
lora_path=lora_path,
parallel_config=DiffusionParallelConfig(ring_degree=ring_degree, allgather_degree=allgather_degree),
)
def test_v2_release_has_nine_sigma_nodes_and_eight_intervals():
assert FASTH3_V2_BASE_SCHEDULE.base_schedule == (
0.999,
0.874,
0.749,
0.624,
0.5,
0.375,
0.25,
0.125,
0.0,
)
assert len(FASTH3_V2_BASE_SCHEDULE.base_schedule) == 9
assert FASTH3_V2_BASE_SCHEDULE.num_inference_steps == 8
assert FASTH3_V2_BASE_SCHEDULE.shifted_sigmas(10.0) == pytest.approx(
[
0.9998999099189271,
0.985788405143244,
0.9675752486758817,
0.94316807738815,
0.9090909090909091,
0.8571428571428571,
0.7692307692307693,
0.5882352941176471,
0.0,
],
abs=1e-12,
)
assert FASTH3_V2_BASE_SCHEDULE.shifted_sigmas(3.0) == pytest.approx(
[
0.9996664442961973,
0.9541484716157204,
0.8995196156925539,
0.8327402135231315,
0.75,
0.6428571428571429,
0.5,
0.3,
0.0,
],
abs=1e-12,
)
def test_fastvideo_metadata_builds_release_sampling_metadata():
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
assert spec.vsa_sparsity == 0.8
assert spec.release_metadata() == {
"partition": "fl2va",
"tasks": ["t2va"],
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
"base_schedule": list(FASTH3_V2_BASE_SCHEDULE.base_schedule),
}
@pytest.mark.parametrize(
("field", "value"),
[
("model_id", "FastVideo/FastH3-other"),
("schema_version", "fasth3-inference-contract-v0"),
("dmd_denoising_steps", [999, 749, 500, 250]),
("transformer_forwards", 4),
("video_scheduler_shift", 12.0),
("audio_scheduler_shift", 4.0),
("guidance_scale", 2.0),
("attention_backend", "FLASH_ATTN"),
("vsa_sparsity", 0.9),
("vsa_tile_size", 32),
("task", "t2va"),
],
)
def test_fastvideo_metadata_rejects_contract_drift(field, value):
metadata = _fastvideo_metadata()
metadata[field] = value
with pytest.raises(ValueError, match=field):
FastH3CheckpointSpec.from_metadata(metadata)
def test_fastvideo_metadata_rejects_missing_contract_field():
metadata = _fastvideo_metadata()
metadata.pop("task")
with pytest.raises(ValueError, match="task"):
FastH3CheckpointSpec.from_metadata(metadata)
def test_legacy_h3_metadata_is_not_treated_as_fastvideo_release():
with pytest.raises(ValueError, match="model_id"):
FastH3CheckpointSpec.from_metadata(
{
"schema_version": 1,
"partition": "fl2va",
"tasks": ["t2va", "fl2va"],
"sigma_shift_scales": {"video": 12.0, "audio": 3.0},
}
)
def test_v2_serving_contract_accepts_backend_selected_for_h3_self_role():
config = _od_config(
backend="TORCH_SDPA",
per_role={"self": AttentionSpec(backend="FASTVIDEO_VSA")},
)
FastH3CheckpointSpec.from_metadata(_fastvideo_metadata()).check_serving_contract(
partition="fl2va", od_config=config
)
@pytest.mark.parametrize(
("kwargs", "partition", "match"),
[
({}, "ref2va", "task-type fl2va"),
({"lora_path": "/tmp/adapter"}, "fl2va", "LoRA"),
({"backend": "TORCH_SDPA"}, "fl2va", "FASTVIDEO_VSA"),
({"topk": 2}, "fl2va", "fixed fastvideo_vsa_topk"),
({"ring_degree": 2}, "fl2va", "local attention"),
({"allgather_degree": 2}, "fl2va", "local attention"),
],
)
def test_v2_serving_contract_rejects_incompatible_runtime(kwargs, partition, match):
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
with pytest.raises(ValueError, match=match):
spec.check_serving_contract(partition=partition, od_config=_od_config(**kwargs))
def test_v2_serving_contract_reads_a_per_role_fixed_topk_override():
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
config = _od_config(
per_role={"self": AttentionSpec(backend="FASTVIDEO_VSA", fastvideo_vsa_topk=1)},
)
with pytest.raises(ValueError, match="fixed fastvideo_vsa_topk"):
spec.check_serving_contract(partition="fl2va", od_config=config)
def test_v2_request_contract_accepts_omitted_steps_and_default_shifts():
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
spec.check_request(_sampling(num_inference_steps=None))
spec.check_request(_sampling(extra_args={"flow_shift": 10, "audio_flow_shift": 3}, guidance_scale=1))
@pytest.mark.parametrize(
"sampling",
[
_sampling(num_inference_steps=None),
_sampling(timesteps=torch.tensor([0.9, 0.5, 0.0])),
_sampling(sigmas=[1.0, 0.5, 0.0]),
],
)
def test_v2_step_execution_requires_pinned_count_and_schedule(sampling):
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
with pytest.raises(OmniClientError, match="step execution"):
spec.check_request(sampling, step_execution=True)
def test_v2_step_execution_accepts_only_explicit_pinned_count():
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
spec.check_request(_sampling(num_inference_steps=8), step_execution=True)
@pytest.mark.parametrize(
("kwargs", "match"),
[
({"num_inference_steps": 7}, "num_inference_steps=8"),
({"num_inference_steps": 9}, "num_inference_steps=8"),
({"num_inference_steps": "8"}, "num_inference_steps=8"),
({"extra_args": {"flow_shift": 9}}, "flow_shift=10"),
({"extra_args": {"audio_flow_shift": 4}}, "audio_flow_shift=3"),
({"extra_args": {"flow_shift": "bad"}}, "flow_shift=10"),
({"guidance_scale": 2.0}, "guidance_scale=1"),
(
{"lora_request": LoRARequest(lora_name="test", lora_int_id=1, lora_path="/tmp/test.safetensors")},
"per-request LoRA",
),
],
)
def test_v2_request_contract_rejects_sampling_drift(kwargs, match):
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
with pytest.raises(OmniClientError, match=match):
spec.check_request(_sampling(**kwargs))
def test_resolve_native_vaes_uses_pinned_revision_and_only_vae_patterns(tmp_path, monkeypatch):
model_root = tmp_path / "FastH3"
model_root.mkdir()
revision = "base-revision-123"
(model_root / "provenance.json").write_text(
json.dumps({"base_model": f"hf://{FASTH3_BASE_MODEL}@{revision}"}),
encoding="utf-8",
)
snapshot = tmp_path / "hf-snapshot"
calls: dict[str, object] = {}
def fake_download(**kwargs: object) -> str:
calls.update(kwargs)
snapshot.mkdir()
return str(snapshot)
monkeypatch.setattr(
"vllm_omni.diffusion.models.minimax_h3.fasth3_checkpoint.download_weights_from_hf_specific",
fake_download,
)
result = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata()).resolve_native_vaes(model_root)
assert result == snapshot / "FL2VA"
assert calls == {
"model_name_or_path": FASTH3_BASE_MODEL,
"cache_dir": None,
"allow_patterns": ["FL2VA/video_vae/**", "FL2VA/audio_vae/**"],
"revision": revision,
"require_all": True,
}
assert sorted(path.name for path in model_root.iterdir()) == ["provenance.json"]
assert not (model_root / "model_index.json").exists()
assert not (model_root / "conversion.json").exists()
@pytest.mark.parametrize(
"base_model",
[
"MiniMaxAI/MiniMax-H3@main",
"hf://MiniMaxAI/MiniMax-H3@",
"hf://Other/MiniMax-H3@revision",
"",
None,
],
)
def test_resolve_native_vaes_rejects_unpinned_provenance(tmp_path, base_model):
model_root = tmp_path / "FastH3"
model_root.mkdir()
(model_root / "provenance.json").write_text(json.dumps({"base_model": base_model}), encoding="utf-8")
with pytest.raises(ValueError, match="provenance"):
FastH3CheckpointSpec.from_metadata(_fastvideo_metadata()).resolve_native_vaes(model_root)
def test_modular_local_model_root_uses_text_encoder_without_partition_files(tmp_path):
root = tmp_path / "FastH3"
(root / "text_encoder").mkdir(parents=True)
(root / "text_encoder" / "config.json").write_text("{}", encoding="utf-8")
(root / "modular_model_index.json").write_text(
json.dumps({"_class_name": "MiniMaxH3ModularPipeline"}),
encoding="utf-8",
)
assert is_minimax_h3_modular(str(root)) is True
assert resolve_minimax_h3_model_root(str(root), None, "fl2va") == str(root / "text_encoder")
assert not (root / "FL2VA").exists()
def test_modular_hub_model_root_downloads_only_text_encoder(monkeypatch, tmp_path):
from vllm_omni.model_executor.models.minimax_h3 import checkpoint as checkpoint_module
snapshot = tmp_path / "modular-snapshot"
calls: dict[str, object] = {}
def fake_index(model: str, *, revision: str | None = None) -> dict[str, str]:
assert model == "FastVideo/FastVideo-FastH3-8-Step-V2"
assert revision == "release-revision"
return {"_class_name": "MiniMaxH3ModularPipeline"}
def fake_download(**kwargs: object) -> str:
calls.update(kwargs)
snapshot.mkdir()
return str(snapshot)
monkeypatch.setattr(checkpoint_module, "get_diffusion_model_index", fake_index)
monkeypatch.setattr(checkpoint_module, "download_weights_from_hf_specific", fake_download)
result = resolve_minimax_h3_model_root(
"FastVideo/FastVideo-FastH3-8-Step-V2",
"release-revision",
"fl2va",
)
assert result == str(snapshot / "text_encoder")
assert calls == {
"model_name_or_path": "FastVideo/FastVideo-FastH3-8-Step-V2",
"cache_dir": None,
"allow_patterns": ["text_encoder/**"],
"revision": "release-revision",
"require_all": True,
}
def test_native_hub_model_with_modular_class_keeps_partition_layout(monkeypatch):
from vllm_omni.model_executor.models.minimax_h3 import checkpoint as checkpoint_module
monkeypatch.setattr(
checkpoint_module,
"get_diffusion_model_index",
lambda _model, *, revision=None: {"_class_name": "MiniMaxH3ModularPipeline"},
)
assert checkpoint_module.is_minimax_h3_modular("MiniMaxAI/MiniMax-H3") is False
def test_native_local_model_root_keeps_partition_layout(tmp_path):
root = tmp_path / "MiniMax-H3"
(root / "FL2VA" / "text_encoder").mkdir(parents=True)
(root / "FL2VA" / "text_encoder" / "config.json").write_text("{}", encoding="utf-8")
assert is_minimax_h3_modular(str(root)) is False
assert resolve_minimax_h3_model_root(str(root), None, "fl2va") == str(root / "FL2VA" / "text_encoder")
def test_diffusion_resolver_returns_local_modular_root_without_partition_files(tmp_path):
root = tmp_path / "FastH3"
root.mkdir()
(root / "modular_model_index.json").write_text(
json.dumps({"_class_name": "MiniMaxH3ModularPipeline"}),
encoding="utf-8",
)
assert resolve_minimax_h3_diffusion_model_path(str(root), None, "fl2va") == str(root)
assert not (root / "FL2VA").exists()
assert not (root / "Ref2VA").exists()
def test_diffusion_resolver_hub_modular_downloads_transformer_contract_only(monkeypatch, tmp_path):
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as pipeline_module
snapshot = tmp_path / "modular-snapshot"
calls: dict[str, object] = {}
def fake_modular(_model: str, _revision: str | None = None) -> bool:
return True
def fake_download(**kwargs: object) -> str:
calls.update(kwargs)
snapshot.mkdir()
return str(snapshot)
monkeypatch.setattr(pipeline_module, "is_minimax_h3_modular", fake_modular)
monkeypatch.setattr(pipeline_module, "download_weights_from_hf_specific", fake_download)
result = resolve_minimax_h3_diffusion_model_path(
"FastVideo/FastVideo-FastH3-8-Step-V2", "release-revision", "fl2va"
)
assert result == str(snapshot)
assert calls == {
"model_name_or_path": "FastVideo/FastVideo-FastH3-8-Step-V2",
"cache_dir": None,
"allow_patterns": [
"modular_model_index.json",
"fastvideo_inference.json",
"provenance.json",
"transformer/**",
],
"revision": "release-revision",
"require_all": True,
}
patterns = calls["allow_patterns"]
assert isinstance(patterns, list)
assert all("video_vae" not in pattern and "audio_vae" not in pattern for pattern in patterns)
def test_diffusers_arch_config_normalizes_all_aliases():
arch = MiniMaxH3DiTArchConfig.from_mapping(
{
"num_refiner_layers": 3,
"ffn_dim": 12,
"in_channels": 4,
"audio_in_channels": 6,
"freq_dim": 8,
"time_embed_hidden_dim": 10,
"rope_freq_dim": 5,
"patch_size": [1, 2, 2],
}
)
assert arch.token_refiner_num_layers == 3
assert arch.ffn_hidden_size == 12
assert arch.latents_dim == 4
assert arch.audio_latents_dim == 6
assert arch.timestep_input_dim == 8
assert arch.time_embed_hidden_size == 10
assert arch.rope_inv_freq_len == 5
assert arch.patch_size == (1, 2, 2)
class _WeightTarget(nn.Module):
def __init__(self, shape: tuple[int, ...], loader: Callable[..., None]) -> None:
super().__init__()
self.weight = nn.Parameter(torch.zeros(shape))
self.weight.weight_loader = loader
class _PipelineLoadTransformer(nn.Module):
def __init__(self, loaded: set[str]) -> None:
super().__init__()
self.arch = MiniMaxH3DiTArchConfig(num_layers=1)
self._loaded = loaded
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
list(weights)
return self._loaded
def post_load_weights(self) -> None:
pass
def test_pipeline_load_weights_rejects_missing_fast_h3_compression_gate():
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
nn.Module.__init__(pipeline)
pipeline.transformer = _PipelineLoadTransformer({"blocks.0.attn.qkv_proj.weight"})
pipeline.video_vae = None
pipeline.audio_vae = None
pipeline._fasth3 = None
pipeline._fasth3_checkpoint = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
with pytest.raises(ValueError, match="missing compression gates"):
pipeline.load_weights([("transformer.blocks.0.attn.qkv_proj.weight", torch.zeros(1))])
def _load_fixture(
*, diffusers_weights: bool
) -> tuple[MiniMaxH3DiTModel, list[tuple[str | None, torch.Tensor]], list[tuple[int, torch.Tensor]]]:
model = object.__new__(MiniMaxH3DiTModel)
nn.Module.__init__(model)
model.arch = MiniMaxH3DiTArchConfig(
num_layers=1,
hidden_size=3,
num_attention_heads=2,
attention_head_dim=2,
ffn_hidden_size=6,
rope_inv_freq_len=2,
)
model._diffusers_weights = diffusers_weights
model._rope_theta = 10000.0
qkv_calls: list[tuple[str | None, torch.Tensor]] = []
fc1_calls: list[tuple[int, torch.Tensor]] = []
def qkv_loader(param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str | None = None) -> None:
if shard_id is None:
assert loaded_weight.shape == (12, 3)
param.data.copy_(loaded_weight)
qkv_calls.append((None, loaded_weight.clone()))
return
assert loaded_weight.shape == (4, 3)
assert shard_id in {"q", "k", "v"}
offset = {"q": 0, "k": 4, "v": 8}[shard_id]
param.data[offset : offset + 4].copy_(loaded_weight)
qkv_calls.append((shard_id, loaded_weight.clone()))
def fc1_loader(param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: int) -> None:
assert loaded_weight.shape == (3, 3)
param.data[shard_id * 3 : (shard_id + 1) * 3].copy_(loaded_weight)
fc1_calls.append((shard_id, loaded_weight.clone()))
model.blocks = nn.ModuleList([nn.Module()])
model.blocks[0].attn = nn.Module()
model.blocks[0].attn.qkv_proj = _WeightTarget((12, 3), qkv_loader)
model.blocks[0].mlp = nn.Module()
model.blocks[0].mlp.fc1 = _WeightTarget((6, 3), fc1_loader)
model.rope = nn.Module()
model.rope.register_buffer("inv_freq", torch.zeros(2))
return model, qkv_calls, fc1_calls
def test_diffusers_load_weights_uses_shape_aware_qkv_and_mlp_slices():
model, qkv_calls, fc1_calls = _load_fixture(diffusers_weights=True)
q = torch.arange(12, dtype=torch.float32).reshape(4, 3) + 10
k = torch.arange(12, dtype=torch.float32).reshape(4, 3) + 100
v = torch.arange(12, dtype=torch.float32).reshape(4, 3) + 200
mlp = torch.arange(18, dtype=torch.float32).reshape(6, 3) + 300
loaded = model.load_weights(
[
("transformer_blocks.0.attn.to_q.weight", q),
("transformer_blocks.0.attn.to_k.weight", k),
("transformer_blocks.0.attn.to_v.weight", v),
("transformer_blocks.0.ff.net.0.proj.weight", mlp),
]
)
assert loaded == {
"blocks.0.attn.qkv_proj.weight",
"blocks.0.mlp.fc1.weight",
"rope.inv_freq",
}
assert [shard_id for shard_id, _ in qkv_calls] == ["q", "k", "v"]
assert [shard_id for shard_id, _ in fc1_calls] == [0, 1]
torch.testing.assert_close(model.blocks[0].attn.qkv_proj.weight, torch.cat((q, k, v)))
torch.testing.assert_close(model.blocks[0].mlp.fc1.weight, torch.cat((mlp[3:], mlp[:3])))
torch.testing.assert_close(
model.rope.inv_freq,
1.0 / (10000.0 ** (torch.arange(0, 4, 2, dtype=torch.float32) / 4)),
rtol=0,
atol=0,
)
def test_native_load_weights_keeps_grouped_qkv_and_gate_first_layouts():
model, qkv_calls, fc1_calls = _load_fixture(diffusers_weights=False)
grouped_qkv = torch.arange(36, dtype=torch.float32).reshape(12, 3)
fc1 = torch.arange(18, dtype=torch.float32).reshape(6, 3)
loaded = model.load_weights(
[
("blocks.0.attn.qkv_proj.weight", grouped_qkv),
("blocks.0.mlp.fc1.weight", fc1),
]
)
expected_qkv = torch.cat(
(
grouped_qkv[:2],
grouped_qkv[6:8],
grouped_qkv[2:4],
grouped_qkv[8:10],
grouped_qkv[4:6],
grouped_qkv[10:],
)
)
assert loaded == {"blocks.0.attn.qkv_proj.weight", "blocks.0.mlp.fc1.weight"}
assert [shard_id for shard_id, _ in qkv_calls] == [None]
assert [shard_id for shard_id, _ in fc1_calls] == [0, 1]
torch.testing.assert_close(model.blocks[0].attn.qkv_proj.weight, expected_qkv)
torch.testing.assert_close(model.blocks[0].mlp.fc1.weight, fc1)
assert torch.equal(model.rope.inv_freq, torch.zeros(2))
def test_diffusers_load_weights_rejects_unknown_name():
model, _, _ = _load_fixture(diffusers_weights=True)
with pytest.raises(ValueError, match="unsupported Diffusers H3 weight"):
model.load_weights([("transformer_blocks.0.attn.not_a_projection.weight", torch.zeros(4, 3))])
def test_diffusers_load_weights_rejects_duplicate_source_name():
model, _, _ = _load_fixture(diffusers_weights=True)
weight = torch.zeros(4, 3)
with pytest.raises(ValueError, match="duplicate Diffusers H3 weight"):
model.load_weights(
[
("transformer_blocks.0.attn.to_q.weight", weight),
("transformer_blocks.0.attn.to_q.weight", weight),
]
)
def test_diffusers_load_weights_rejects_incomplete_qkv_group():
model, _, _ = _load_fixture(diffusers_weights=True)
with pytest.raises(ValueError, match="incomplete Diffusers H3 QKV"):
model.load_weights(
[
("transformer_blocks.0.attn.to_q.weight", torch.zeros(4, 3)),
("transformer_blocks.0.attn.to_k.weight", torch.zeros(4, 3)),
]
)
def test_diffusers_load_weights_rejects_missing_compression_gate_parameter():
model, _, _ = _load_fixture(diffusers_weights=True)
with pytest.raises(ValueError, match="has no model parameter"):
model.load_weights([("transformer_blocks.0.attn.to_gate_compress.weight", torch.zeros(4, 3))])
def test_resolve_sigma_positions_uses_release_metadata_schedule():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
nn.Module.__init__(pipeline)
pipeline._fasth3 = None
pipeline._lora_sigma_schedules = {}
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
pipeline._base_schedule_by_partition = {
"fl2va": DMD2SigmaSchedule.from_metadata(spec.release_metadata()),
}
positions, steps = pipeline._resolve_sigma_positions("t2va", _sampling())
assert positions == FASTH3_V2_BASE_SCHEDULE.base_schedule
assert steps == 8 == len(positions) - 1
with pytest.raises(OmniClientError, match="must be 8"):
pipeline._resolve_sigma_positions("t2va", _sampling(num_inference_steps=9))
@@ -87,7 +87,8 @@ def _collective_worker(rank, rendezvous, backend):
sampling = OmniDiffusionSamplingParams(
height=32, width=32, num_frames=96, extra_args={"task": "t2va", "aspect_ratio": "1:1"}
)
text_conditioning = model._prepare_local_conditioning({"prompt": "test"}, sampling)
text_conditioning, window_text = model._prepare_local_conditioning({"prompt": "test"}, sampling)
assert window_text is None
torch.testing.assert_close(text_conditioning.hidden_states, torch.full_like(text_conditioning.hidden_states, 2))
assert text_conditioning.visual_condition is None
@@ -0,0 +1,231 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from vllm_omni.diffusion.models.minimax_h3.continuation import (
diffuse_continuation,
plan_continuation_windows,
resolve_continuation,
)
from vllm_omni.diffusion.models.minimax_h3.packed_sequence import minimax_h3_packed_sequence_ref2va_blocks
from vllm_omni.errors import OmniClientError
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@pytest.mark.parametrize("vary_prompts", [False, True])
@pytest.mark.parametrize("total,window", [(1807, 277), (1807, 243), (124, 277)])
def test_continuation_retains_prefix_and_global_stereo_timeline(total, window, vary_prompts):
audio_t = round(total * 40 / 24)
source = torch.arange(2 * audio_t * 32).reshape(2, audio_t, 32).float()
calls = []
plan = plan_continuation_windows(total, window, 22)
texts = [(torch.full((3 + i, 8), float(i)), torch.ones(3 + i, dtype=torch.long)) for i in range(len(plan))]
def sample(**kw):
calls.append(kw)
t = kw["latent_t"]
if vary_prompts:
assert kw["text_embeddings"] is texts[len(calls) - 1][0]
assert kw["text_tags"] is texts[len(calls) - 1][1]
assert kw["media_time_origin"] == len(texts[-1][0])
# Different constants expose replacement of an old prefix or an
# accidental append of the hidden overlap.
video = torch.full((1, 24, t, 2, 2), float(len(calls)))
audio = kw["locked_audio_rows"].reshape(2, kw["audio_t"], 32).permute(0, 2, 1)
if len(calls) > 1:
assert torch.all(kw["visual_condition"] == len(calls) - 1)
assert kw["ref_blocks"][-1]["kind"] == "latent_guide"
return video, audio
kwargs = dict(
num_frames=total,
latent_t=(total - 5) // 17 * 5 + 2,
latent_h=2,
latent_w=2,
audio_t=audio_t,
locked_audio_rows=source.reshape(-1, 32),
)
video, audio = diffuse_continuation(
sample, kwargs, window_frames=window, overlap_frames=22, text_conditioning=texts if vary_prompts else None
)
torch.testing.assert_close(audio, source.permute(0, 2, 1))
plan = plan_continuation_windows(total, window, 22)
assert [call["temporal_offset"] for call in calls] == pytest.approx([part.start * 40 / 24 for part in plan])
offset = 0
for i, part in enumerate(plan):
new_frames = part.end - part.start - part.overlap
count = (new_frames - 5) // 17 * 5 + 2 if i == 0 else new_frames // 17 * 5
assert torch.all(video[:, :, offset : offset + count] == i + 1)
offset += count
assert offset == video.shape[2] == kwargs["latent_t"]
@pytest.mark.parametrize("temporal_offset", [0.0, 425.0, 221 * 40 / 24])
def test_latent_guide_shares_target_clock_and_remains_condition_only(temporal_offset):
packed = minimax_h3_packed_sequence_ref2va_blocks(
temporal_offset=temporal_offset,
text_len=3,
latent_t=12,
latent_h=4,
latent_w=4,
audio_t=68,
ref_blocks=[
{"kind": "image", "latent_h": 4, "latent_w": 4},
{"kind": "latent_guide", "latent_t": 7, "latent_h": 4, "latent_w": 4, "ref_audio_t": 37},
],
)
pos = packed["img_position_ids"]
visual = packed["img_pos"]
target = visual[packed["update_mask"]]
guide = visual[~packed["update_mask"]][4:]
torch.testing.assert_close(pos[guide], pos[target[:28]])
audio = packed["audio_pos"]
guide_audio = audio[~packed["audio_update_mask"]].reshape(2, 37)
target_audio = audio[packed["audio_update_mask"]].reshape(2, 68)
torch.testing.assert_close(pos[guide_audio], pos[target_audio[:, :37]])
assert pos[target[0], 0] == pytest.approx(4 + temporal_offset)
torch.testing.assert_close(pos[:3, 0], torch.arange(3, dtype=torch.float64))
assert torch.all(pos[3:7, 0] == 3) # static reference image stays fixed
@pytest.mark.parametrize(
"extra,step",
[
({"long_video_mode": "bad"}, False),
({"long_video_mode": "continuation"}, True),
({"long_video_mode": "continuation", "continuation_overlap_frames": 23}, False),
({"long_video_mode": "continuation", "continuation_window_frames": True}, False),
({"long_video_mode": "continuation", "continuation_overlap_frames": 277}, False),
],
)
def test_invalid_continuation_options(extra, step):
with pytest.raises(OmniClientError):
resolve_continuation(extra, task="ref2va", step_execution=step)
def test_global_offset_moves_temporal_references_but_not_images_or_padding():
kwargs = dict(
text_len=3,
latent_t=12,
latent_h=4,
latent_w=4,
audio_t=68,
ref_blocks=[
{"kind": "image", "latent_h": 4, "latent_w": 4},
{"kind": "audio", "ref_audio_t": 3},
{"kind": "video_audio", "ref_audio_t": 2, "latent_t": 2, "latent_h": 4, "latent_w": 4},
{"kind": "image", "latent_h": 4, "latent_w": 4},
{"kind": "latent_guide", "ref_audio_t": 37, "latent_t": 7, "latent_h": 4, "latent_w": 4},
],
)
base = minimax_h3_packed_sequence_ref2va_blocks(**kwargs)
shifted = minimax_h3_packed_sequence_ref2va_blocks(**kwargs, temporal_offset=425.0)
moving = base["image_mask"] | base["audio_mask"]
moving[3:7] = False # first static image
moving[25:29] = False # static image after temporal references
delta = shifted["img_position_ids"] - base["img_position_ids"]
torch.testing.assert_close(delta[:, 0], moving.double() * 425)
assert torch.count_nonzero(delta[:, 1:]) == 0
for key, value in base.items():
if isinstance(value, torch.Tensor) and key != "img_position_ids":
torch.testing.assert_close(shifted[key], value)
def test_long_ref2va_defaults_to_continuation_with_explicit_full_opt_out():
assert resolve_continuation({"long_video": True}, task="ref2va", step_execution=False) == (277, 22)
assert resolve_continuation({}, task="ref2va", step_execution=False) is None
assert (
resolve_continuation({"long_video": True, "long_video_mode": "full"}, task="ref2va", step_execution=False)
is None
)
def test_different_chunk_prompt_lengths_keep_the_same_media_clock():
kwargs = dict(
latent_t=7,
latent_h=4,
latent_w=4,
audio_t=37,
ref_blocks=[{"kind": "image", "latent_h": 4, "latent_w": 4}],
media_time_origin=20,
)
first = minimax_h3_packed_sequence_ref2va_blocks(text_len=3, **kwargs)
second = minimax_h3_packed_sequence_ref2va_blocks(text_len=15, temporal_offset=425, **kwargs)
for key, mask in (("img_pos", "update_mask"), ("audio_pos", "audio_update_mask")):
a = first["img_position_ids"][first[key][first[mask]]]
b = second["img_position_ids"][second[key][second[mask]]]
torch.testing.assert_close(b[:, 0] - a[:, 0], torch.full_like(a[:, 0], 425))
torch.testing.assert_close(b[:, 1:], a[:, 1:])
@pytest.mark.parametrize("prompt_count", [None, 7, 6])
def test_request_encodes_only_window_prompts_and_prepares_references_once(monkeypatch, prompt_count):
from PIL import Image
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3EncoderMediaConditioning
pipeline = object.__new__(module.MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.load_text_encoder = True
pipeline.device = torch.device("cpu")
pipeline.od_config = None
pipeline._active_turbo_spec = lambda sampling: None
pipeline._has_active_native_lora = lambda sampling: False
pipeline._resolve_task = lambda *args, **kwargs: "ref2va"
encoded, prepared_inputs, encoded_media = [], [], []
prepare = module.prepare_encoder_inputs
def prepare_once(*args, **kwargs):
result = prepare(*args, **kwargs)
prepared_inputs.append(result)
return result
def encode_text(prepared):
encoded.append(prepared)
return torch.full((3, 5120), len(encoded), dtype=torch.bfloat16), torch.zeros(3, dtype=torch.long)
def encode_media(media):
encoded_media.append(media)
return MiniMaxH3EncoderMediaConditioning(
task=media.task,
height=media.height,
width=media.width,
num_frames=media.num_frames,
latent_t=media.latent_t,
audio_t=media.audio_t,
)
monkeypatch.setattr(module, "prepare_encoder_inputs", prepare_once)
pipeline.encode_prompt = encode_text
pipeline._encode_local_media = encode_media
pipeline._prepare_encoder_conditioning_inputs = lambda conditioning, sampling: {
"text_embeddings": conditioning.hidden_states,
"continuation": (277, 22),
"num_frames": conditioning.num_frames,
"task": conditioning.task,
}
extra = {"task": "ref2va", "long_video": True, "duration": 75}
if prompt_count is not None:
extra["continuation_prompts"] = [f"window {i}" for i in range(prompt_count)]
sampling = OmniDiffusionSamplingParams(width=64, height=64, fps=24, extra_args=extra)
request = {"prompt": "main prompt", "multi_modal_data": {"image": Image.new("RGB", (256, 256))}}
if prompt_count == 6:
with pytest.raises(OmniClientError, match="exactly 7"):
pipeline._prepare_request_inputs(request, sampling)
assert not encoded and not encoded_media
return
context = pipeline._prepare_request_inputs(request, sampling)
assert [item.prompt for item in encoded] == extra.get("continuation_prompts", ["main prompt"])
assert len(prepared_inputs) == len(encoded_media) == 1
assert all(item.media is prepared_inputs[0].media for item in encoded)
if prompt_count is not None:
texts = context["continuation_text_conditioning"]
assert len(texts) == 7
assert texts[0][0] is context["text_embeddings"]
assert [int(hidden[0, 0]) for hidden, _ in texts] == list(range(1, 8))
else:
assert "continuation_text_conditioning" not in context
@@ -3,7 +3,7 @@
import json
import sys
from contextlib import contextmanager
from contextlib import contextmanager, nullcontext
from multiprocessing.reduction import ForkingPickler
from types import SimpleNamespace
from typing import Any, TypeVar
@@ -282,15 +282,44 @@ def test_encoder_task_validation_and_transformer_routing():
assert pipeline._resolve_task(None, {"image": object()}) == "fl2va"
assert pipeline._resolve_task(None, {"audio": object()}) == "ref2va"
assert pipeline._resolve_task(None, {"video": object()}) == "ref2va"
assert pipeline._resolve_task(None, {"audio": object()}, audio_mode="lock_source") == "t2va"
assert (
pipeline._resolve_task(
None,
{"image": object(), "audio": object()},
audio_mode="lock_source",
)
== "fl2va"
)
assert (
pipeline._resolve_task(
None,
{"video": object(), "audio": object()},
audio_mode="lock_source",
)
== "ref2va"
)
fl2v_spec = _turbo_spec("minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors")
ref2v_spec = _turbo_spec("minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors")
assert pipeline._resolve_task("fl2va", {"image": object()}, turbo_spec=fl2v_spec) == "fl2va"
with pytest.raises(OmniClientError, match="serves"):
pipeline._resolve_task("ref2va", {}, turbo_spec=fl2v_spec)
assert pipeline._resolve_task("ref2va", {"image": object()}, turbo_spec=ref2v_spec) == "ref2va"
pipeline.partition = "fl2va"
pipeline.supported_tasks = frozenset({"t2va", "fl2va"})
assert pipeline._resolve_task(None, {"audio": object()}, audio_mode="lock_source") == "t2va"
assert (
pipeline._resolve_task(
None,
{"image": object(), "audio": object()},
audio_mode="lock_source",
)
== "fl2va"
)
pipeline.partition = "ref2va"
pipeline.supported_tasks = frozenset({"ref2va"})
assert pipeline._resolve_task("ref2va") == "ref2va"
assert pipeline._resolve_task(None, {"audio": object()}, audio_mode="lock_source") == "ref2va"
pipeline.partition = "combined"
pipeline.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
@@ -423,6 +452,7 @@ def test_pipeline_loads_task_selected_components_with_encoder_ownership(
assert pipeline.partition == expected_partition
assert pipeline.supported_tasks == expected_tasks
assert len(created["dit"]) == expected_dits
assert all(kwargs["diffusers_weights"] is False for _, kwargs in created["dit"])
component_path = tmp_path / component_partition
assert created["video_vae"] == [str(component_path / "video_vae")]
assert created["audio_vae"] == [str(component_path / "audio_vae")]
@@ -1041,6 +1071,56 @@ def test_packed_attention_rejects_backends_without_multi_doc_capability(backend_
)
def test_rainfusion_packed_padding_stays_mask_free_on_unaligned_lengths():
"""Regression for the #5543 follow-up crash fixed in #7235.
MiniMax-H3 pads every packed request to a multiple of 64 rows, so any
non-64-aligned length reaches this path with used < packed_total. Before
RAINFUSION_ATTN advertised supports_prefix_kv_slicing, the model built a
padding mask here that _assert_metadata_compatible then rejected at the
first sparse layer. This drives the real backend class (not a fake flag)
through the packed path; both the sparse dispatch and the dense fallback
share this metadata construction and neither ever reads attn_mask.
"""
from vllm_omni.diffusion.attention.backends.rainfusion_attn import (
RainFusionAttentionBackend,
)
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
MiniMaxH3Attention,
)
class FakeAttention(torch.nn.Module):
attn_backend = RainFusionAttentionBackend
def __init__(self):
super().__init__()
self.metadata = None
def forward(self, query, key, value, metadata):
self.metadata = metadata
return query
attention = object.__new__(MiniMaxH3Attention)
torch.nn.Module.__init__(attention)
attention.attention = FakeAttention()
q = torch.randn(8, 2, 4)
attention._run_packed_attention(
q,
q,
q,
# One request: 5 valid rows padded to the 8-row alignment boundary.
cu_seqlens=torch.tensor([0, 5, 8], dtype=torch.int32),
max_seqlen=5,
packed_total=8,
)
metadata = attention.attention.metadata
assert metadata is not None
assert metadata.attn_mask is None
assert metadata.extra["valid_kv_length"] == 5
def test_reference_image_resize_contract():
from PIL import Image
@@ -1099,6 +1179,7 @@ def test_minimax_h3_advertises_the_official_ref2va_image_limit():
from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata
assert get_diffusion_model_metadata("MiniMaxH3Pipeline").max_multimodal_image_inputs == 9
assert get_diffusion_model_metadata("MiniMaxH3Pipeline").supports_latent_mask_editing
def test_text_attention_routes_local_gqa_heads_through_sdpa_helper(monkeypatch):
@@ -2756,41 +2837,63 @@ def test_dit_encoder_selection_keeps_vae_resident():
component.offload_to_cpu.assert_not_called()
def test_keyframe_encode_pins_and_restores_cudnn_settings():
@pytest.mark.parametrize(
("capability", "allow_tf32"),
[((8, 6), True), ((9, 0), False), ((10, 0), True), ((10, 3), True), (None, True)],
)
@pytest.mark.parametrize("initial_tf32", [False, True])
@pytest.mark.parametrize("fail_encode", [False, True])
def test_keyframe_encode_pins_and_restores_cudnn_settings(
monkeypatch, capability, allow_tf32, initial_tf32, fail_encode
):
from vllm.platforms.interface import DeviceCapability
from vllm_omni.diffusion.models.minimax_h3.vae import (
_minimax_h3_keyframe_encode_context,
)
from vllm_omni.platforms import current_omni_platform
def get_capability(device_id):
assert device_id == 1
return DeviceCapability(*capability) if capability is not None else None
monkeypatch.setattr(current_omni_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_omni_platform, "get_device_capability", get_capability)
cudnn = torch.backends.cudnn
original = (
cudnn.enabled,
cudnn.benchmark,
cudnn.deterministic,
cudnn.allow_tf32,
)
try:
cudnn.enabled = False
cudnn.benchmark = True
cudnn.deterministic = False
cudnn.allow_tf32 = False
with _minimax_h3_keyframe_encode_context(torch.device("cuda")):
with cudnn.flags(enabled=False, benchmark=True, deterministic=False, allow_tf32=initial_tf32):
error_context = pytest.raises(RuntimeError, match="encode failed") if fail_encode else nullcontext()
with error_context, _minimax_h3_keyframe_encode_context(torch.device("cuda:1")):
assert cudnn.enabled
assert not cudnn.benchmark
assert cudnn.deterministic
assert cudnn.allow_tf32
assert cudnn.allow_tf32 is allow_tf32
if fail_encode:
raise RuntimeError("encode failed")
assert not cudnn.enabled
assert cudnn.benchmark
assert not cudnn.deterministic
assert not cudnn.allow_tf32
finally:
(
cudnn.enabled,
cudnn.benchmark,
cudnn.deterministic,
cudnn.allow_tf32,
) = original
assert cudnn.allow_tf32 is initial_tf32
def test_keyframe_encode_cpu_keeps_cudnn_settings(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3.vae import (
_minimax_h3_keyframe_encode_context,
)
from vllm_omni.platforms import current_omni_platform
def unexpected_capability_query(device_id):
pytest.fail("CPU keyframe encode must not query CUDA capability")
monkeypatch.setattr(current_omni_platform, "get_device_capability", unexpected_capability_query)
cudnn = torch.backends.cudnn
with cudnn.flags(enabled=False, benchmark=True, deterministic=False, allow_tf32=False):
with _minimax_h3_keyframe_encode_context(torch.device("cpu")):
assert not cudnn.enabled
assert cudnn.benchmark
assert not cudnn.deterministic
assert not cudnn.allow_tf32
@pytest.mark.parametrize("fail_encode", [False, True])
@@ -2851,15 +2954,17 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
class FakeEncoder:
instances = []
instances: list["FakeEncoder"] = []
def __init__(self, **kwargs):
self.pushes = []
self.kwargs = kwargs
self.__class__.instances.append(self)
def push(self, frames):
def push(self, frames, *, on_consumed=None):
self.pushes.append(np.array(frames, copy=True))
if on_consumed is not None:
on_consumed()
def finish(self):
return b"mp4"
@@ -2872,6 +2977,8 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
return torch.zeros(1, 1, 2)
class FakeVideoVAE:
chunk_value_range = (0.0, 1.0)
def decode_with_chunks(self, latent, *, on_chunk):
for value in (0.0, 0.25, 0.5):
on_chunk(torch.full((1, 3, 3, 2, 2), value))
@@ -2883,7 +2990,7 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
pipeline.device = torch.device("cpu")
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
monkeypatch.setattr(
"vllm_omni.diffusion.utils.media_utils.ChunkedMP4Encoder",
"vllm_omni.diffusion.utils.chunked_video.ChunkedMP4Encoder",
FakeEncoder,
)
@@ -2903,19 +3010,90 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
assert encoder.pushes[1].shape == (3, 2, 2, 3)
def _peer_rank_preencode_pipeline(monkeypatch):
"""An H3 pipeline whose video VAE is a distributed rank that owns no output.
Every rank of the VAE group must call ``decode_with_chunks`` to keep the
temporal collectives in lockstep, but only the owner receives chunks, so a
peer's callback is never invoked.
"""
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
class FakeAudioVAE:
def decode_latent(self, latent):
return torch.zeros(1, 1, 2)
class PeerRankVideoVAE:
chunk_value_range = (0.0, 1.0)
def decode_with_chunks(self, latent, *, on_chunk):
del latent, on_chunk
pipeline = object.__new__(mod.MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.audio_vae = FakeAudioVAE()
pipeline.video_vae = PeerRankVideoVAE()
pipeline.device = torch.device("cpu")
pipeline.od_config = SimpleNamespace()
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
return pipeline
def test_peer_vae_rank_returns_no_preencoded_output_instead_of_failing(monkeypatch):
"""A rank that owns no output must not turn its silence into a request error."""
pipeline = _peer_rank_preencode_pipeline(monkeypatch)
output = pipeline.decode_to_mp4(torch.zeros(1), torch.zeros(1), height=2, width=2)
assert output == b""
def test_peer_vae_rank_post_decode_reaches_post_processing(monkeypatch):
"""The real H3 post_decode chain, not just the shared consumer, tolerates a peer rank."""
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
from vllm_omni.diffusion.worker.utils import StepRequestState
pipeline = _peer_rank_preencode_pipeline(monkeypatch)
monkeypatch.setattr(pipeline, "_unpack_denoised_rows", lambda *args, **kwargs: (torch.zeros(1), torch.zeros(1)))
state = StepRequestState(
request_id="peer-rank",
sampling=SimpleNamespace(num_outputs_per_prompt=1),
prompt="a prompt",
)
state.latents = torch.zeros(1)
state.extra[mod._STEP_BRANCH] = object()
state.extra[mod._STEP_AUDIO_ROWS] = torch.zeros(1)
state.extra[mod._STEP_SHAPE] = {
"latent_t": 1,
"latent_h": 1,
"latent_w": 1,
"audio_t": 1,
"height": 2,
"width": 2,
"preencode_mp4": True,
}
output = pipeline.post_decode(state)
assert output.output == (b"", None)
assert mod._minimax_h3_post_process(output.output)["video"] == [b""]
def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatch):
"""A client's encoder options must survive the worker-side pre-encode path."""
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
class FakeEncoder:
instances = []
instances: list["FakeEncoder"] = []
def __init__(self, **kwargs):
self.kwargs = kwargs
self.__class__.instances.append(self)
def push(self, frames):
def push(self, frames, *, on_consumed=None):
del frames
if on_consumed is not None:
on_consumed()
def finish(self):
return b"mp4"
@@ -2928,6 +3106,8 @@ def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatc
return torch.zeros(1, 1, 2)
class FakeVideoVAE:
chunk_value_range = (0.0, 1.0)
def decode_with_chunks(self, latent, *, on_chunk):
on_chunk(torch.zeros(1, 3, 1, 2, 2))
@@ -2937,7 +3117,7 @@ def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatc
pipeline.video_vae = FakeVideoVAE()
pipeline.device = torch.device("cpu")
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
monkeypatch.setattr("vllm_omni.diffusion.utils.media_utils.ChunkedMP4Encoder", FakeEncoder)
monkeypatch.setattr("vllm_omni.diffusion.utils.chunked_video.ChunkedMP4Encoder", FakeEncoder)
pipeline.decode_to_mp4(
torch.zeros(1),
@@ -3010,3 +3190,19 @@ def test_video_codec_options_are_normalized_for_the_encoder():
assert normalize_video_codec_options(None) is None
with pytest.raises(ValueError, match="video_codec_options"):
normalize_video_codec_options("preset=ultrafast")
@pytest.mark.parametrize("duration", [60, 75])
def test_long_video_shape_requires_explicit_opt_in(duration):
from vllm_omni.errors import OmniClientError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import resolve_minimax_h3_shape
sampling = OmniDiffusionSamplingParams(width=960, height=544, fps=24, extra_args={"duration": duration})
with pytest.raises(OmniClientError, match="15"):
resolve_minimax_h3_shape("ref2va", sampling, None)
sampling.extra_args["long_video"] = True
_, _, frames, video_t, audio_t = resolve_minimax_h3_shape("ref2va", sampling, None)
assert frames >= duration * 24 and frames % 17 == 5
assert video_t == (frames - 5) // 17 * 5 + 2
assert audio_t == round(frames / 24 * 40)
@@ -262,7 +262,10 @@ def test_a_vsa_variant_is_recognised_by_its_compression_gates(tmp_path):
def test_a_tensor_naming_no_h3_parameter_is_an_error(tmp_path):
path = tmp_path / "fasth3" / "adapter_model.safetensors"
_write_adapter(path, tensors={"transformer_blocks.0.attn.norm_q.lora_A.weight": torch.ones((_RANK, _HIDDEN))})
_write_adapter(
path,
tensors={"transformer_blocks.0.attn.not_a_projection.lora_A.weight": torch.ones((_RANK, _HIDDEN))},
)
# Dropping it silently would load a model that is not the distilled student.
with pytest.raises(FastH3AdapterError, match="name no known"):
_load(path.parent)
@@ -0,0 +1,259 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Pipeline-level MiniMax H3 latent initialization tests."""
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
import pytest
import torch
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
def test_prepare_edit_video_clones_the_last_frame_with_lossless_rgb(monkeypatch, tmp_path):
from vllm_omni.model_executor.models.minimax_h3 import reference_video
source = tmp_path / "source.webm"
commands: list[list[str]] = []
monkeypatch.setattr(
reference_video,
"_probe_video",
lambda _path: {"frame_count": 12, "audio_codecs": ("opus",)},
)
monkeypatch.setattr(
reference_video.subprocess,
"run",
lambda command, **_kwargs: commands.append(command),
)
prepared = reference_video.prepare_edit_video(
source,
target_width=448,
target_height=256,
target_frame_count=73,
workdir=str(tmp_path / "prepared"),
)
assert prepared["input_has_audio"] is True
assert prepared["frame_count"] == 73
command = commands[0]
assert command[command.index("-vf") + 1].endswith("tpad=stop_mode=clone:stop=-1")
assert command[command.index("-c:v") + 1] == "libx264rgb"
assert command[command.index("-pix_fmt") + 1] == "rgb24"
@pytest.mark.parametrize("source_audio", [None, "override.wav"])
def test_encoder_preparation_routes_edit_sources_without_qwen_inputs(
monkeypatch,
tmp_path,
source_audio,
):
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing as processing
latent_t, audio_t = 7, 37
video_clean = torch.full((42, 96), 3.0)
audio_source = torch.zeros(4, 32)
audio_source[:, 0] = torch.tensor([1.0, 2.0, 11.0, 12.0])
waveform = torch.zeros(3_200)
frames = torch.zeros(22, 64, 96, 3, dtype=torch.uint8).numpy()
calls: dict[str, Any] = {}
def prepare_video(value, **kwargs):
calls["prepare"] = (value, kwargs)
return {
"original_path": "source.mov",
"prepared_path": str(tmp_path / "prepared.mp4"),
"input_has_audio": True,
}
def load_video_audio(value, **kwargs):
calls["video_audio"] = (value, kwargs)
return waveform, 32_000
def load_audio_file(value):
calls["audio_file"] = value
return waveform, 32_000
class VideoVAE:
def encode_video(self, value):
calls["video_encode_shape"] = value.shape
return video_clean.clone(), (latent_t, 4, 6)
class AudioVAE:
def encode_waveform(self, value, sample_rate):
calls["audio_encode"] = (value.shape, sample_rate)
return audio_source.clone(), 2
monkeypatch.setattr(processing, "resolve_minimax_h3_shape", lambda *_args: (64, 96, 22, latent_t, audio_t))
monkeypatch.setattr(processing, "prepare_edit_video", prepare_video)
monkeypatch.setattr(processing, "load_video_frames", lambda _path: frames)
monkeypatch.setattr(processing, "load_video_audio", load_video_audio)
monkeypatch.setattr(processing, "load_audio_file", load_audio_file)
prepared = processing.prepare_encoder_inputs(
{
"prompt": "edit the clip",
"multi_modal_data": {
"source_video": "source.mov",
"source_audio": source_audio,
"video_noise_mask": torch.zeros(1, 1, latent_t, 4, 6),
"audio_noise_mask": torch.tensor([0.0, 0.5, *([0.0] * (audio_t - 2))]).reshape(1, 1, -1),
},
},
SimpleNamespace(extra_args={"task": "t2va"}),
)
media = type(prepared.media).from_mm_tensors(
prepared.media.to_mm_tensors(),
prepared.media.to_metadata(),
)
conditioning = processing.encode_media(
media,
video_vae=VideoVAE(),
audio_vae=AudioVAE(),
emit_conditioning=True,
)
assert prepared.images == []
assert prepared.qwen_videos == []
assert prepared.condition_labels == []
assert prepared.media.images == ()
assert prepared.media.videos == ()
assert calls["prepare"][0] == "source.mov"
assert calls["prepare"][1]["target_width"] == 96
assert calls["prepare"][1]["target_height"] == 64
assert calls["prepare"][1]["target_frame_count"] == 22
if source_audio is None:
assert calls["video_audio"] == (
"source.mov",
{"duration_seconds": pytest.approx(22 / 24)},
)
assert "audio_file" not in calls
else:
assert calls["audio_file"] == "override.wav"
assert "video_audio" not in calls
assert conditioning is not None
torch.testing.assert_close(conditioning.video_edit_clean_rows, video_clean)
assert conditioning.video_edit_mask.shape == (latent_t, 4, 6)
torch.testing.assert_close(conditioning.video_edit_mask, torch.zeros(latent_t, 4, 6))
assert conditioning.audio_edit_source_t == 2
assert conditioning.audio_edit_mask.shape == (2, audio_t)
torch.testing.assert_close(
conditioning.audio_edit_clean_rows[:, 0],
torch.tensor([1.0, 2.0, *([0.0] * (audio_t - 2)), 11.0, 12.0, *([0.0] * (audio_t - 2))]),
)
torch.testing.assert_close(
conditioning.audio_edit_mask,
torch.tensor([0.0, 0.5, *([0.0] * (audio_t - 2))]).repeat(2, 1),
)
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
MiniMaxH3EncoderConditioning,
MiniMaxH3TextConditioning,
)
combined = MiniMaxH3EncoderConditioning.from_components(
MiniMaxH3TextConditioning(
torch.zeros(1, 5120, dtype=torch.bfloat16),
torch.ones(1, dtype=torch.int64),
),
conditioning,
)
round_trip = MiniMaxH3EncoderConditioning.from_omni_payload(combined.to_omni_payload())
torch.testing.assert_close(round_trip.video_edit_clean_rows, video_clean)
torch.testing.assert_close(round_trip.video_edit_mask, torch.zeros(latent_t, 4, 6))
torch.testing.assert_close(round_trip.audio_edit_clean_rows, conditioning.audio_edit_clean_rows)
torch.testing.assert_close(
round_trip.audio_edit_mask,
torch.tensor([0.0, 0.5, *([0.0] * (audio_t - 2))]).repeat(2, 1),
)
assert round_trip.audio_edit_source_t == 2
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.default_video_shift = 12.0
pipeline.default_audio_shift = 3.0
pipeline.od_config = SimpleNamespace(step_execution=False)
pipeline._fasth3 = None
pipeline._active_turbo_spec = lambda _sampling: None
pipeline._has_active_native_lora = lambda _sampling: False
pipeline._resolve_task = lambda task, **_kwargs: task
pipeline._resolve_sigma_positions = lambda _task, _sampling: (None, 3)
pipeline._quality_policy = SimpleNamespace(resolve=lambda **_kwargs: SimpleNamespace(cache_dit=None))
pipeline._cache_dit_runtime = SimpleNamespace(prepare=lambda _plan: None)
context = pipeline._prepare_encoder_conditioning_inputs(
round_trip,
SimpleNamespace(extra_args={}, quality=None, seed=17, num_outputs_per_prompt=1),
)
torch.testing.assert_close(context["video_edit_clean_rows"], video_clean)
torch.testing.assert_close(context["video_edit_mask_rows"], torch.zeros(42))
assert context["video_edit_restore_mask_rows"].shape == (42, 96)
torch.testing.assert_close(
context["audio_edit_mask_rows"],
torch.tensor([0.0, 0.5, *([1.0] * (audio_t - 2))]).repeat(2),
)
torch.testing.assert_close(context["audio_edit_mask_rows"], context["audio_edit_restore_mask_rows"])
def _build_inputs(*, video_mask: torch.Tensor, audio_mask: torch.Tensor):
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
latent_t, latent_h, latent_w, audio_t = 2, 4, 6, 3
video_clean = torch.linspace(-1.0, 1.0, latent_t * (latent_h // 2) * (latent_w // 2) * 96).reshape(-1, 96)
audio_clean = torch.linspace(-0.5, 0.5, audio_t * 2 * 32).reshape(-1, 32)
inputs = pipeline._build_denoise_inputs(
task="t2va",
text_embeddings=torch.zeros(2, 4),
text_tags=torch.ones(2, dtype=torch.long),
seed=17,
latent_t=latent_t,
latent_h=latent_h,
latent_w=latent_w,
audio_t=audio_t,
num_frames=22,
num_steps=3,
video_shift=12.0,
audio_shift=3.0,
base_schedule=None,
visual_condition=None,
visual_condition_shape=None,
audio_condition=None,
ref_audio_t=None,
video_edit_clean_rows=video_clean,
video_edit_mask_rows=video_mask,
video_edit_restore_mask_rows=video_mask,
audio_edit_clean_rows=audio_clean,
audio_edit_mask_rows=audio_mask,
audio_edit_restore_mask_rows=audio_mask,
)
return pipeline, inputs, video_clean, audio_clean
def test_build_denoise_inputs_uses_h3_source_anchors():
pipeline, inputs, video_clean, audio_clean = _build_inputs(
video_mask=torch.zeros(12),
audio_mask=torch.zeros(6),
)
video_edit = inputs["video_edit"]
audio_edit = inputs["audio_edit"]
assert video_edit is not None and audio_edit is not None
initial_video, _ = pipeline._initial_noise(
seed=17,
latent_t=2,
latent_h=4,
latent_w=6,
audio_t=3,
)
torch.testing.assert_close(video_edit.clean_rows, video_clean)
torch.testing.assert_close(video_edit.anchor_rows, 0.999 * video_clean + 0.001 * initial_video)
torch.testing.assert_close(audio_edit.clean_rows, audio_clean)
torch.testing.assert_close(audio_edit.anchor_rows, audio_clean)
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Focused CPU tests for MiniMax H3 latent-mask editing."""
from __future__ import annotations
import pytest
import torch
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
def test_video_full_grid_quantization_and_cell_level_restore():
from vllm_omni.diffusion.models.minimax_h3.latent_mask import minimax_h3_video_edit_masks
shape = dict(latent_t=2, latent_h=4, latent_w=6)
full = torch.arange(48, dtype=torch.float32).reshape(2, 4, 6) / 48
cells = full.reshape(2, 2, 2, 3, 2).permute(0, 1, 3, 2, 4)
parsed = minimax_h3_video_edit_masks(full, **shape)
expected_model = torch.ceil(cells.amax(dim=(3, 4)).reshape(-1) * 256) / 256
expected_restore = cells.unsqueeze(3).expand(2, 2, 3, 24, 2, 2).reshape(12, 96)
torch.testing.assert_close(parsed.model_mask_rows, expected_model)
torch.testing.assert_close(parsed.restore_mask_rows, expected_restore)
def test_audio_grid_uses_channel_major_order_and_preserves_raw_values():
from vllm_omni.diffusion.models.minimax_h3.latent_mask import minimax_h3_audio_edit_masks
grid = torch.tensor([[0.0, 0.1, 1.0], [0.25, 0.5, 0.75]])
expected = grid.flatten()
parsed = minimax_h3_audio_edit_masks(grid, audio_t=3)
torch.testing.assert_close(parsed.model_mask_rows, torch.ceil(expected * 256) / 256)
torch.testing.assert_close(parsed.restore_mask_rows, expected)
def test_zero_fractional_one_semantics_and_cell_level_restore():
from vllm_omni.diffusion.models.minimax_h3.latent_mask import (
MiniMaxH3LatentEdit,
minimax_h3_video_edit_masks,
)
clean = torch.full((3, 2), 10.0)
anchor = torch.tensor([[8.0, 8.0], [6.0, 6.0], [4.0, 4.0]])
state = torch.tensor([[2.0, 2.0], [4.0, 4.0], [6.0, 6.0]])
velocity = torch.full((3, 2), 2.0)
mask = torch.tensor([0.0, 0.5, 1.0])
edit = MiniMaxH3LatentEdit.from_rows(clean, anchor, mask, mask)
assert edit is not None
model_rows, target_timesteps = edit.prepare(state, 0.25, 0.999, sigma=0.75)
torch.testing.assert_close(model_rows, torch.lerp(anchor, state, mask[:, None]))
torch.testing.assert_close(
target_timesteps,
torch.tensor([0.999, 0.625, 0.25]),
)
predicted = model_rows + 0.75 * velocity * mask[:, None]
torch.testing.assert_close(edit.x0(model_rows, velocity, 0.25), torch.lerp(clean, predicted, mask[:, None]))
# A single active cell drives the full model token, while raw restoration
# preserves the other three cells independently in every packed channel.
parsed = minimax_h3_video_edit_masks(
torch.tensor([[[1.0, 0.0], [0.0, 0.0]]]),
latent_t=1,
latent_h=2,
latent_w=2,
)
restore = torch.tensor([1.0, 0.0, 0.0, 0.0]).repeat(24).reshape(1, 96)
torch.testing.assert_close(parsed.model_mask_rows, torch.ones(1))
torch.testing.assert_close(parsed.restore_mask_rows, restore)
cell_edit = MiniMaxH3LatentEdit.from_rows(
torch.full((1, 96), 10.0),
torch.full((1, 96), 8.0),
parsed.model_mask_rows,
parsed.restore_mask_rows,
)
assert cell_edit is not None
cell_state = torch.full((1, 96), 2.0)
cell_velocity = torch.full((1, 96), 4.0)
expected = torch.where(restore.bool(), cell_state + 0.75 * cell_velocity, cell_edit.clean_rows)
torch.testing.assert_close(cell_edit.x0(cell_state, cell_velocity, 0.25), expected)
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Duration limits apply before encoding and to transported encoder payloads."""
from types import SimpleNamespace
from typing import Any
import pytest
import torch
from PIL import Image
from vllm_omni.errors import OmniClientError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import (
prepare_encoder_inputs,
resolve_minimax_h3_shape,
)
from vllm_omni.model_executor.models.minimax_h3.long_video import validate_encoded_frame_limit
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@pytest.mark.parametrize("mode,limit", [("full", 30), ("continuation", 300), (None, 300)])
@pytest.mark.parametrize("input_kind", ["duration", "duration_seconds", "target", "num_frames"])
def test_long_video_limits_cover_all_duration_inputs(mode, limit, input_kind):
extra: dict[str, Any] = {"long_video": True}
if mode is not None:
extra["long_video_mode"] = mode
sampling = OmniDiffusionSamplingParams(width=64, height=64, fps=24, extra_args=extra)
def set_duration(seconds):
if input_kind == "target":
extra["target"] = {"duration_seconds": seconds}
elif input_kind == "num_frames":
sampling.num_frames = seconds * 24
else:
extra[input_kind] = seconds
set_duration(limit)
_, _, frames, _, _ = resolve_minimax_h3_shape("ref2va", sampling, None)
assert frames >= limit * 24
validate_encoded_frame_limit(extra, "ref2va", frames)
with pytest.raises(OmniClientError, match="request limit"):
validate_encoded_frame_limit(extra, "ref2va", frames + 17)
set_duration(limit + 1)
with pytest.raises(OmniClientError, match="duration"):
resolve_minimax_h3_shape("ref2va", sampling, None)
@pytest.mark.parametrize("task", ["t2va", "fl2va", "ref2va"])
def test_full_mode_rejects_300_seconds_and_accepts_comparison(task):
sampling = OmniDiffusionSamplingParams(
width=64,
height=64,
fps=24,
extra_args={"long_video": True, "long_video_mode": "full", "duration": 300, "aspect_ratio": "1:1"},
)
image = Image.new("RGB", (64, 64)) if task == "fl2va" else None
with pytest.raises(OmniClientError, match="30"):
resolve_minimax_h3_shape(task, sampling, image)
sampling.extra_args["duration"] = 20
assert resolve_minimax_h3_shape(task, sampling, image)[2] == 481
@pytest.mark.parametrize("mode", ["full", "continuation"])
def test_mode_alone_does_not_bypass_long_video_opt_in(mode):
sampling = OmniDiffusionSamplingParams(
width=64,
height=64,
extra_args={"long_video_mode": mode, "duration": 20},
)
with pytest.raises(OmniClientError, match="15"):
resolve_minimax_h3_shape("ref2va", sampling, None)
@pytest.mark.parametrize("kind", ["video", "audio"])
def test_continuation_edit_rejected_before_loading_source(kind):
sampling = OmniDiffusionSamplingParams(
width=64,
height=64,
extra_args={"long_video": True, "task": "ref2va", "duration": 20},
)
with pytest.raises(OmniClientError, match="continuation does not support latent-mask editing"):
prepare_encoder_inputs(
{
"prompt": "test",
"multi_modal_data": {
"image": Image.new("RGB", (64, 64)),
f"source_{kind}": "must-not-be-loaded",
f"{kind}_noise_mask": 0.5,
},
},
sampling,
)
def test_locked_audio_rejects_audio_edit_before_loading_source():
sampling = OmniDiffusionSamplingParams(
width=64,
height=64,
extra_args={"task": "t2va", "aspect_ratio": "1:1", "audio_mode": "lock_source"},
)
with pytest.raises(OmniClientError, match="cannot be combined"):
prepare_encoder_inputs(
{
"prompt": "test",
"multi_modal_data": {
"audio": (torch.zeros(8000), 8000),
"source_audio": "must-not-be-loaded",
"audio_noise_mask": 0.5,
},
},
sampling,
)
@pytest.mark.parametrize("kind", ["video", "audio"])
def test_external_encoder_cannot_bypass_continuation_edit_validation(kind):
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(step_execution=False)
conditioning = SimpleNamespace(
task="ref2va",
num_frames=481,
video_edit_clean_rows=torch.zeros(1) if kind == "video" else None,
audio_edit_clean_rows=torch.zeros(1) if kind == "audio" else None,
)
sampling = OmniDiffusionSamplingParams(extra_args={"long_video": True})
with pytest.raises(OmniClientError, match="continuation does not support latent-mask editing"):
pipeline._prepare_encoder_conditioning_inputs(conditioning, sampling)
def test_external_encoder_cannot_bypass_full_duration_limit():
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(step_execution=False)
sampling = OmniDiffusionSamplingParams(extra_args={"long_video": True, "long_video_mode": "full"})
with pytest.raises(OmniClientError, match="30-second request limit"):
pipeline._prepare_encoder_conditioning_inputs(SimpleNamespace(task="ref2va", num_frames=7213), sampling)
@@ -190,3 +190,31 @@ def test_fused_modulation_preserves_bf16_residual_boundary() -> None:
)
assert torch.equal(modulated_out, expected)
@pytest.mark.cuda
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="CUDA required")
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
@pytest.mark.parametrize("fused_gate", [False, True])
def test_fused_modulation_keeps_fp32_affine_intermediates(fused_gate: bool) -> None:
device = current_omni_platform.get_torch_device()
rows, hidden_size = 2, 5376
x = torch.ones(rows, hidden_size, device=device, dtype=torch.bfloat16)
weight = torch.full((hidden_size,), 1.125, device=device, dtype=x.dtype)
shift = torch.zeros(1, hidden_size, device=device, dtype=x.dtype)
scale = torch.full_like(shift, 0.09375)
indices = torch.zeros(rows, device=device, dtype=torch.int64)
eps = 1e-6
if fused_gate:
residual, actual = indexed_gate_rms_norm_scale_shift(
x, torch.zeros_like(shift), x, weight, shift, scale, indices, eps
)
assert torch.equal(residual, x)
else:
actual = rms_norm_indexed_scale_shift(x, weight, shift, scale, indices, eps)
# Without the RMS epsilon, 1.125 * 1.09375 = 1.23046875 is exactly
# halfway between two BF16 values. FP32 normalization keeps the epsilon
# and rounds down to 1.2265625. Premature BF16 rounding drops the epsilon
# and instead rounds the tie up to 1.234375, changing every output row.
assert torch.equal(actual, torch.full_like(x, 1.2265625))
@@ -86,6 +86,7 @@ def test_decode_releases_only_audio_before_return(monkeypatch):
audio_vae=Mock(decode_latent=Mock(return_value=audio)),
_component_on_device=lambda component: nullcontext(),
_offload_model_cpu_stage_output=Mock(side_effect=lambda value: value),
_release_stage_cache=Mock(),
)
monkeypatch.setattr(
module.current_omni_platform,
@@ -152,6 +153,7 @@ def test_forward_releases_video_after_quantization_and_before_next_seed(monkeypa
diffuse=diffuse,
decode=decode,
_offload_model_cpu_stage_output=_recording_release(events, releases),
_release_stage_cache=lambda: None,
)
request = SimpleNamespace(prompts=["a prompt"], sampling_params=SimpleNamespace())
@@ -201,6 +203,7 @@ def test_forward_drops_the_decoded_video_before_the_next_seed_diffuses(monkeypat
diffuse=diffuse,
decode=decode,
_offload_model_cpu_stage_output=lambda value: value.clone(),
_release_stage_cache=lambda: None,
)
request = SimpleNamespace(prompts=["a prompt"], sampling_params=SimpleNamespace())
@@ -231,6 +234,7 @@ def test_post_decode_releases_video_after_quantization(monkeypatch):
_unpack_denoised_rows=lambda *args, **kwargs: (torch.zeros(1), torch.zeros(1)),
decode=decode,
_offload_model_cpu_stage_output=_recording_release(events, releases),
_release_stage_cache=lambda: None,
)
state = SimpleNamespace(
latents=torch.zeros(1),
@@ -529,7 +533,7 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
torch.nn.Module.__init__(pipeline)
pipeline.audio_vae = Mock()
pipeline._model_cpu_offload_modules = [pipeline.audio_vae]
events = []
events: list[tuple] = []
@contextmanager
def record_component(value):
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
from unittest.mock import Mock
@@ -41,7 +42,7 @@ class _FakeAttention(nn.Module):
super().__init__()
def _small_od_config():
def _small_od_config(*, class_name=None):
arch = {
"num_layers": 1,
"token_refiner_num_layers": 1,
@@ -60,6 +61,8 @@ def _small_od_config():
"final_adaln_out_features": 2 * 8,
"rope_inv_freq_len": 2,
}
if class_name is not None:
arch["_class_name"] = class_name
return SimpleNamespace(
tf_model_config=arch,
parallel_config=SimpleNamespace(ulysses_degree=1),
@@ -133,6 +136,24 @@ def test_fp8_scope_and_prefix_propagation(monkeypatch):
)
def test_explicit_native_weight_format_overrides_diffusers_class_name(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import minimax_h3_transformer as h3
monkeypatch.setattr(h3, "ColumnParallelLinear", _FakeLinear)
monkeypatch.setattr(h3, "MergedColumnParallelLinear", _FakeLinear)
monkeypatch.setattr(h3, "QKVParallelLinear", _FakeLinear)
monkeypatch.setattr(h3, "RowParallelLinear", _FakeLinear)
monkeypatch.setattr(h3, "Attention", _FakeAttention)
monkeypatch.setattr(h3, "get_tensor_model_parallel_world_size", lambda: 1)
model = h3.MiniMaxH3DiTModel(
_small_od_config(class_name="MiniMaxH3Transformer3DModel"),
diffusers_weights=False,
)
assert model._diffusers_weights is False
class _WeightTarget(nn.Module):
def __init__(self, loader):
super().__init__()
@@ -187,8 +187,14 @@ def test_minimax_h3_quantization_quality(config: QualityTestConfig):
"model": model_ref,
"enforce_eager": True,
# The fused BF16 baseline only fits on H100-80GB with encoder TP.
"stage_0_tensor_parallel_size": 2,
# This test loads the direct FL2VA diffusion pipeline (not the
# disaggregated two-stage MiniMax-H3 pipeline), so configure both the
# DiT world and the Qwen3-VL encoder TP explicitly.
"tensor_parallel_size": 2,
"text_encoder_tp_size": 2,
"vae_use_tiling": True,
# MiniMax H3's VAE is implemented by code shipped with the checkpoint.
"trust_remote_code": True,
}
omni_bl = Omni(**common_kwargs)
@@ -27,6 +27,9 @@ class _SegmentMeanModel:
assignment shows up as a numeric difference instead of passing silently.
"""
def __init__(self):
self.calls: list[dict[str, torch.Tensor]] = []
def __call__(self, **kwargs):
x = kwargs["x"][0]
audio_x = kwargs["audio_x"][0]
@@ -45,6 +48,14 @@ class _SegmentMeanModel:
pooled_audio[start:stop] = audio_x[start:stop].mean(dim=0, keepdim=True)
row_timesteps = kwargs["unique_timesteps"][kwargs["inverse_indices"]].unsqueeze(-1)
self.calls.append(
{
"video_rows": x[img_pos].clone(),
"audio_rows": audio_x[audio_pos].clone(),
"video_timesteps": row_timesteps[img_pos, 0].clone(),
"audio_timesteps": row_timesteps[audio_pos, 0].clone(),
}
)
video = (pooled_video + 0.5 * x + row_timesteps)[img_pos]
audio = (pooled_audio + 0.5 * audio_x + row_timesteps)[audio_pos]
if not kwargs.get("skip_mask_out_condition", False):
@@ -156,9 +167,77 @@ def test_step_execution_matches_request_mode_denoise_loop():
torch.testing.assert_close(state.extra[mod._STEP_AUDIO_ROWS], reference_audio)
def test_batched_step_execution_matches_independent_requests():
def test_step_execution_matches_request_mode_with_latent_edits():
"""Masked model rows, row timesteps, and scheduler math match both paths."""
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
from vllm_omni.diffusion.models.minimax_h3.denoise_loop import minimax_h3_denoise_loop
from vllm_omni.diffusion.models.minimax_h3.latent_mask import MiniMaxH3LatentEdit
model = _SegmentMeanModel()
branch, video_rows, audio_rows = _make_branch(text_len=9, latent_t=2, latent_h=4, latent_w=6, audio_t=3, seed=51)
sigmas_video = _sigmas(5, 12.0)
sigmas_audio = _sigmas(5, 3.0)
video_clean = torch.linspace(-1.0, 1.0, video_rows.numel()).reshape_as(video_rows)
audio_clean = torch.linspace(1.0, -1.0, audio_rows.numel()).reshape_as(audio_rows)
video_mask = torch.linspace(0.0, 1.0, video_rows.shape[0])
video_edit = MiniMaxH3LatentEdit.from_rows(
video_clean,
0.999 * video_clean + 0.001 * video_rows,
video_mask,
video_mask,
)
audio_mask = torch.tensor([0.0, 0.2, 0.4, 0.6, 0.8, 1.0])
audio_edit = MiniMaxH3LatentEdit.from_rows(
audio_clean,
audio_clean,
audio_mask,
audio_mask,
)
reference_video, reference_audio = minimax_h3_denoise_loop(
model=model,
positive=branch,
initial_video_rows=video_rows,
initial_audio_rows=audio_rows,
keyframe_cond_rows=None,
video_edit=video_edit,
audio_edit=audio_edit,
sigmas_video=sigmas_video,
sigmas_audio=sigmas_audio,
device=torch.device("cpu"),
)
request_first_call = model.calls[0]
torch.testing.assert_close(request_first_call["video_rows"], video_edit.model_rows(video_rows))
torch.testing.assert_close(request_first_call["audio_rows"], audio_edit.model_rows(audio_rows))
torch.testing.assert_close(
request_first_call["video_timesteps"],
video_edit.target_timesteps(1.0 - sigmas_video[0], 0.999, sigma=sigmas_video[0]),
)
torch.testing.assert_close(
request_first_call["audio_timesteps"],
audio_edit.target_timesteps(1.0 - sigmas_audio[0], 1.0, sigma=sigmas_audio[0]),
)
model.calls.clear()
state = _make_state("req-edit", model, branch, video_rows, audio_rows, sigmas_video, sigmas_audio)
state.extra[mod._STEP_VIDEO_EDIT] = video_edit
state.extra[mod._STEP_AUDIO_EDIT] = audio_edit
pipeline = _step_pipeline(model)
while not state.denoise_completed:
velocity = pipeline.denoise_step(SimpleNamespace(states=(state,)), states=[state])
pipeline.step_scheduler(state, velocity)
for name, expected in request_first_call.items():
torch.testing.assert_close(model.calls[0][name], expected)
torch.testing.assert_close(state.latents, reference_video)
torch.testing.assert_close(state.extra[mod._STEP_AUDIO_ROWS], reference_audio)
@pytest.mark.parametrize("lock_audio", [False, True])
def test_batched_step_execution_matches_independent_requests(lock_audio):
"""Two co-batched requests must land where they would have landed alone."""
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
from vllm_omni.diffusion.models.minimax_h3.latent_mask import MiniMaxH3LatentEdit
model = _SegmentMeanModel()
specs = [
@@ -171,20 +250,40 @@ def test_batched_step_execution_matches_independent_requests():
pipeline = _step_pipeline(model)
def make_state(request_id: str, index: int):
branch, video_rows, audio_rows = _make_branch(**specs[index])
sigmas_video, sigmas_audio = schedules[index]
state = _make_state(request_id, model, branch, video_rows, audio_rows, sigmas_video, sigmas_audio)
video_mask = torch.linspace(0.1 * index, 0.8 + 0.1 * index, video_rows.shape[0])
video_edit = MiniMaxH3LatentEdit.from_rows(
torch.full_like(video_rows, 10.0 + index),
torch.full_like(video_rows, 20.0 + index),
video_mask,
video_mask,
)
audio_mask = torch.linspace(0.2 - 0.1 * index, 1.0 - 0.1 * index, audio_rows.shape[0])
audio_edit = MiniMaxH3LatentEdit.from_rows(
torch.full_like(audio_rows, 30.0 + index),
torch.full_like(audio_rows, 40.0 + index),
audio_mask,
audio_mask,
)
state.extra[mod._STEP_VIDEO_EDIT] = video_edit
if lock_audio and index == 0:
branch.locked_audio_rows = audio_rows.clone()
else:
state.extra[mod._STEP_AUDIO_EDIT] = audio_edit
return state
alone: list[tuple[torch.Tensor, torch.Tensor]] = []
for spec, (sigmas_video, sigmas_audio) in zip(specs, schedules):
branch, video_rows, audio_rows = _make_branch(**spec)
state = _make_state("solo", model, branch, video_rows, audio_rows, sigmas_video, sigmas_audio)
for index in range(len(specs)):
state = make_state("solo", index)
while not state.denoise_completed:
pipeline.step_scheduler(state, pipeline.denoise_step(SimpleNamespace(states=(state,)), states=[state]))
alone.append((state.latents, state.extra[mod._STEP_AUDIO_ROWS]))
states = []
for index, (spec, (sigmas_video, sigmas_audio)) in enumerate(zip(specs, schedules)):
branch, video_rows, audio_rows = _make_branch(**spec)
if index == 0:
assert branch.used_len == branch.seq_len
states.append(_make_state(f"req-{index}", model, branch, video_rows, audio_rows, sigmas_video, sigmas_audio))
states = [make_state(f"req-{index}", index) for index in range(len(specs))]
assert states[0].extra[mod._STEP_BRANCH].used_len == states[0].extra[mod._STEP_BRANCH].seq_len
active = list(states)
while active:
@@ -291,6 +390,8 @@ def test_prepare_encode_seeds_runner_visible_state(monkeypatch, batch_frames):
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
branch, video_rows, audio_rows = _make_branch(text_len=9, latent_t=2, latent_h=4, latent_w=6, audio_t=3, seed=8)
video_edit = object()
audio_edit = object()
sigmas_video = _sigmas(6, 12.0)
sigmas_audio = _sigmas(6, 3.0)
context = {
@@ -326,6 +427,8 @@ def test_prepare_encode_seeds_runner_visible_state(monkeypatch, batch_frames):
"audio_rows": audio_rows,
"cond_anchor": None,
"audio_anchor": None,
"video_edit": video_edit,
"audio_edit": audio_edit,
"sigmas_video": sigmas_video,
"sigmas_audio": sigmas_audio,
},
@@ -347,6 +450,8 @@ def test_prepare_encode_seeds_runner_visible_state(monkeypatch, batch_frames):
assert state.do_true_cfg is False
torch.testing.assert_close(state.current_timestep, torch.tensor(1.0 - sigmas_video[0]))
assert state.extra[mod._STEP_BRANCH] is branch
assert state.extra[mod._STEP_VIDEO_EDIT] is video_edit
assert state.extra[mod._STEP_AUDIO_EDIT] is audio_edit
assert state.extra[mod._STEP_SHAPE]["height"] == 96
pipeline.od_config = SimpleNamespace()
@@ -450,3 +555,33 @@ def test_packed_batch_rejects_backends_that_cannot_isolate_requests(attention):
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
assert MiniMaxH3Pipeline._packed_batch_supported(_FakeTransformer([attention])) is False
def test_locked_driving_audio_is_clean_and_unchanged_during_denoising():
from vllm_omni.diffusion.models.minimax_h3.denoise_loop import minimax_h3_denoise_loop
branch, video, audio = _make_branch(text_len=3, latent_t=2, latent_h=2, latent_w=2, audio_t=3, seed=7)
branch.locked_audio_rows = audio.clone()
seen = []
def model(**kwargs):
positions = kwargs["audio_pos_info"]["position_ids"]
times = kwargs["unique_timesteps"][kwargs["inverse_indices"]]
torch.testing.assert_close(times[positions], torch.ones_like(times[positions]))
torch.testing.assert_close(kwargs["audio_x"][0, positions], audio)
seen.append(True)
return torch.ones_like(video), torch.ones_like(audio)
result_video, result_audio = minimax_h3_denoise_loop(
model=model,
positive=branch,
initial_video_rows=video,
initial_audio_rows=audio,
keyframe_cond_rows=None,
sigmas_video=[1.0, 0.5, 0.0],
sigmas_audio=[1.0, 0.25, 0.0],
device=torch.device("cpu"),
)
assert len(seen) == 2
torch.testing.assert_close(result_audio, audio)
assert not torch.equal(result_video, video)
@@ -0,0 +1,139 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU tests for in-place decode revert and stage-boundary cache release.
``_revert_decoded_inplace`` must be numerically identical to the checkpoint's
``revert_tensor`` (denormalize → clamp → rearrange) with zero whole-video
copies, and ``_release_stage_cache`` must force the allocator-cache release
(or fall back to a plain empty_cache when no component cache exists).
"""
import pytest
import torch
import vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 as pipeline_module
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.diffusion.models.minimax_h3.vae import MiniMaxH3VideoVAE
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
DENORM_MEAN = (-2.11628412, -2.03571429, -1.80444444) # -imagenet_mean/std
DENORM_STD = (4.36681223, 4.46428571, 4.44444444) # 1/imagenet_std
class _FakeDenormalize:
def __init__(self, mean, std):
self.mean = mean
self.std = std
class _FakeProcessor:
def __init__(self):
self.transform_rev = _FakeDenormalize(DENORM_MEAN, DENORM_STD)
self.use_3d_conv = True
self.revert_calls = 0
def revert_tensor(self, tensor):
self.revert_calls += 1
return _legacy_revert(tensor)
class _FakeCheckpointModel:
def __init__(self):
self.processor = _FakeProcessor()
def _vae(model=None):
vae = object.__new__(MiniMaxH3VideoVAE)
vae.model = model if model is not None else _FakeCheckpointModel()
return vae
def _legacy_revert(tensor: torch.Tensor) -> torch.Tensor:
"""The checkpoint's revert_tensor, op for op, on the CPU (B == 1)."""
if tensor.ndim == 4:
tensor = tensor.unsqueeze(2)
assert tensor.ndim == 5
# Normalize runs on the (b t) c h w rearrangement; at B == 1 that is a
# view of (B, C, T, H, W), so the per-channel broadcast is equivalent.
mean = torch.tensor(DENORM_MEAN).view(1, 3, 1, 1, 1)
std = torch.tensor(DENORM_STD).view(1, 3, 1, 1, 1)
tensor = (tensor - mean) / std
return tensor.clamp(0, 1)
def test_revert_inplace_matches_legacy_bitwise():
decoded = torch.randn(1, 3, 7, 6, 8).clamp_(-3, 3)
expected = _legacy_revert(decoded.clone())
out = _vae()._revert_decoded_inplace(decoded.clone())
assert out.shape == expected.shape == (1, 3, 7, 6, 8)
assert torch.equal(out, expected)
def test_revert_inplace_mutates_without_extra_copies():
decoded = torch.randn(1, 3, 5, 4, 4)
out = _vae()._revert_decoded_inplace(decoded)
assert out is decoded # in-place: no whole-video copy materializes
assert out.is_contiguous()
def test_revert_inplace_unsqueezes_4d_input():
# 4-D decoded tensors follow the (B, C, H, W) image contract: unsqueeze(2)
# lands T=1, matching the checkpoint's revert_tensor branch.
decoded = torch.randn(1, 3, 6, 8)
expected = _legacy_revert(decoded.clone())
out = _vae()._revert_decoded_inplace(decoded.clone())
assert torch.equal(out, expected)
def test_revert_inplace_falls_back_without_contract():
vae = _vae()
vae.model.processor.transform_rev = type("Empty", (), {})()
decoded = torch.randn(1, 3, 5, 4, 4)
out = vae._revert_decoded_inplace(decoded)
assert vae.model.processor.revert_calls == 1
assert torch.equal(out, _legacy_revert(decoded))
def test_revert_inplace_falls_back_on_bad_arity():
vae = _vae()
vae.model.processor.transform_rev = _FakeDenormalize((0.5, 0.5), (1.0, 1.0, 1.0))
vae._revert_decoded_inplace(torch.randn(1, 3, 5, 4, 4))
assert vae.model.processor.revert_calls == 1
class _RecordingCache:
def __init__(self):
self.calls = []
def release_if_needed(self, *, force=False):
self.calls.append(force)
return True
def _pipeline():
pipeline = object.__new__(MiniMaxH3Pipeline)
pipeline._dlo_component_cache = None
return pipeline
def test_stage_cache_release_forces_component_cache():
pipeline = _pipeline()
cache = _RecordingCache()
pipeline._dlo_component_cache = cache
pipeline._release_stage_cache()
assert cache.calls == [True]
def test_stage_cache_release_falls_back_to_empty_cache(monkeypatch):
released = []
monkeypatch.setattr(
pipeline_module.current_omni_platform,
"empty_cache",
lambda: released.append(True),
raising=True,
)
pipeline = _pipeline()
pipeline._release_stage_cache()
assert released == [True]
@@ -0,0 +1,243 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU tests for the streaming uint8 preparation in the MiniMax-H3 video VAE.
``_stream_prepare_video_tensor`` must be numerically identical to the
checkpoint's whole-video numpy path (``convert_numpy_to_tensor`` →
``transform_tensor`` → ``transpose``) while padding on the host, so the
device-side copies never materialize.
"""
import numpy as np
import pytest
import torch
from vllm_omni.diffusion.models.minimax_h3.vae import MiniMaxH3VideoVAE
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225)
CLIP_LENGTH = 4
class _FakeNormalize:
def __init__(self, mean, std):
self.mean = mean
self.std = std
class _FakeProcessor:
# Read by the stream prep via getattr(..., False); annotation only so
# tests can attach it per case.
isolated_last_frame: bool
def __init__(self):
self.transform = _FakeNormalize(IMAGENET_MEAN, IMAGENET_STD)
class _FakeCheckpointModel:
def __init__(self, **overrides):
self.clip_length = CLIP_LENGTH
self.isolated_first_frame = False
self.frame_pre_padding = 0
self.processor = _FakeProcessor()
for key, value in overrides.items():
setattr(self, key, value)
def _vae(model=None):
vae = object.__new__(MiniMaxH3VideoVAE)
vae.model = model if model is not None else _FakeCheckpointModel()
return vae
def _legacy_reference(frames: np.ndarray) -> torch.Tensor:
"""The checkpoint's numpy path, op for op, on the CPU.
convert_numpy_to_tensor: astype(fp32) → permute → ÷255
transform_tensor: Normalize = (x - mean) / std
encode_videos: transpose(0, 1)
"""
tensor = torch.from_numpy(frames.astype(np.float32)).permute(0, 3, 1, 2)
tensor = tensor / 255.0
mean = torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1)
std = torch.tensor(IMAGENET_STD).view(1, 3, 1, 1)
tensor = (tensor - mean) / std
return tensor.transpose(0, 1).contiguous()
def _video(num_frames, height=6, width=8, seed=0):
rng = np.random.default_rng(seed)
return rng.integers(0, 256, size=(num_frames, height, width, 3), dtype=np.uint8)
def _pad_frames(frames: np.ndarray, pad: int) -> np.ndarray:
if not pad:
return frames
return np.concatenate([frames, np.repeat(frames[-1:], pad, axis=0)])
def test_stream_prep_matches_legacy_path_bitwise():
frames = _video(10)
out = _vae()._stream_prepare_video_tensor(frames, torch.device("cpu"))
assert out is not None
assert out.shape == (3, 12, 6, 8) # pad = (0 - 10) % 4 == 2
assert out.dtype == torch.float32
assert torch.equal(out, _legacy_reference(_pad_frames(frames, 2)))
def test_stream_prep_isolated_first_frame_without_last_frame_skips_pad():
# Asymmetric checkpoint (isolated first frame, no isolated last frame):
# trim-stable lengths are k*clip + 0, which never satisfy the
# encode_temporal offset of 1, so the remote re-pads through its own
# whole-video cat regardless. Padding here would only upload frames the
# remote trim discards, so the legacy frame count is kept.
model = _FakeCheckpointModel(isolated_first_frame=True)
frames = _video(10)
out = _vae(model)._stream_prepare_video_tensor(frames, torch.device("cpu"))
assert out is not None
assert out.shape[1] == 10
assert torch.equal(out, _legacy_reference(frames))
def test_stream_prep_symmetric_isolated_frames_pad_to_trim_stable():
# isolated first + last frames: offset == tail % clip == 1, so padding
# to the next trim-stable length makes the remote trim a no-op and keeps
# encode_temporal from re-padding: T=10, clip=4, tail=1 → 3*4+1 == 13.
model = _FakeCheckpointModel(isolated_first_frame=True)
model.processor.isolated_last_frame = True
frames = _video(10)
out = _vae(model)._stream_prepare_video_tensor(frames, torch.device("cpu"))
assert out is not None
assert out.shape[1] == 13
assert out.shape[1] % 4 == 1
assert torch.equal(out, _legacy_reference(_pad_frames(frames, 3)))
def test_stream_prep_uses_processor_align_when_available():
calls = []
class _AlignedProcessor(_FakeProcessor):
def align_video_length(self, video_length, mode="pad", granularity="chunk"):
calls.append((video_length, mode, granularity))
return 5
model = _FakeCheckpointModel()
model.processor = _AlignedProcessor()
frames = _video(10)
out = _vae(model)._stream_prepare_video_tensor(frames, torch.device("cpu"))
assert out is not None
assert calls == [(10, "pad", "chunk")]
assert out.shape[1] == 15
assert torch.equal(out, _legacy_reference(_pad_frames(frames, 5)))
def test_stream_prep_pads_with_last_frame():
frames = _video(2) # pad = (0 - 2) % 4 == 2
out = _vae()._stream_prepare_video_tensor(frames, torch.device("cpu"))
padded = np.concatenate([frames, np.repeat(frames[-1:], 2, axis=0)])
assert torch.equal(out, _legacy_reference(padded))
def test_stream_prep_aligned_input_needs_no_pad():
frames = _video(8) # 8 % 4 == 0
out = _vae()._stream_prepare_video_tensor(frames, torch.device("cpu"))
assert out.shape[1] == 8
assert torch.equal(out, _legacy_reference(frames))
@pytest.mark.parametrize(
"frames",
[
_video(5).astype(np.float32), # not uint8
_video(5)[:, :, :, :1], # single channel
np.zeros((5, 6, 8, 4), dtype=np.uint8), # four channels
np.zeros((5, 6, 8), dtype=np.uint8), # missing channel axis
np.zeros((0, 6, 8, 3), dtype=np.uint8), # empty
],
)
def test_stream_prep_rejects_unsupported_inputs(frames):
assert _vae()._stream_prepare_video_tensor(frames, torch.device("cpu")) is None
@pytest.mark.parametrize(
"model",
[
_FakeCheckpointModel(clip_length=None),
_FakeCheckpointModel(clip_length=0),
type("NoProcessor", (), {"clip_length": CLIP_LENGTH})(),
type(
"NoTransform",
(),
{"clip_length": CLIP_LENGTH, "processor": type("P", (), {})()},
)(),
],
)
def test_stream_prep_rejects_missing_checkpoint_contract(model):
frames = _video(5)
assert _vae(model)._stream_prepare_video_tensor(frames, torch.device("cpu")) is None
def test_stream_prep_rejects_bad_norm_arity():
model = _FakeCheckpointModel()
model.processor.transform = _FakeNormalize((0.5, 0.5), IMAGENET_STD)
assert _vae(model)._stream_prepare_video_tensor(_video(5), torch.device("cpu")) is None
class _RecordingModel(_FakeCheckpointModel):
"""Feeds encode_video a well-shaped latent and records its input."""
def __init__(self):
super().__init__()
self.received = None
self.latent = torch.randn(1, 4, 3, 4, 4) # (B, C, T', H', W'), even H/W
def encode_videos(self, videos, use_fp16_latent=False, **kwargs):
self.received = videos
assert use_fp16_latent is True
return [self.latent]
def _vae_for_encode_video():
vae = object.__new__(MiniMaxH3VideoVAE)
vae.model = _RecordingModel()
vae.device_module = torch
vae.config_dict = {
"latent_channels": 4,
"latents_mean": [0.0] * 4,
"latents_std": [1.0] * 4,
}
# A plain tensor (not nn.Parameter): nn.Module.__setattr__ refuses
# Parameters on instances built via object.__new__.
vae._dummy_parameter = torch.zeros(1, dtype=torch.float32)
vae.parameters = lambda: iter([vae._dummy_parameter])
return vae
def test_encode_video_streams_uint8_numpy(monkeypatch):
monkeypatch.delenv("VLLM_OMNI_VAE_ENCODE_LEGACY_PREP", raising=False)
vae = _vae_for_encode_video()
rows, shape = vae.encode_video(_video(10))
assert isinstance(vae.model.received, list)
assert isinstance(vae.model.received[0], torch.Tensor)
assert vae.model.received[0].shape == (3, 12, 6, 8)
assert rows.shape[0] == shape[0] * shape[1] * shape[2] // 4 # patchify rows
def test_encode_video_legacy_env_escapes_to_checkpoint_path(monkeypatch):
monkeypatch.setenv("VLLM_OMNI_VAE_ENCODE_LEGACY_PREP", "1")
vae = _vae_for_encode_video()
frames = _video(10)
vae.encode_video(frames)
assert vae.model.received is frames # untouched numpy, checkpoint converts
def test_encode_video_falls_back_when_preparation_unsupported(monkeypatch):
monkeypatch.delenv("VLLM_OMNI_VAE_ENCODE_LEGACY_PREP", raising=False)
vae = _vae_for_encode_video()
vae.model.clip_length = None # contract not discoverable
frames = _video(10)
vae.encode_video(frames)
assert vae.model.received is frames
@@ -0,0 +1,218 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU tests for the split encoder/decoder residency of the MiniMax-H3 VAE.
The encode half (CNN encoder + quant_conv) and the decode half
(post_quant_conv + ViT decoder) stage independently: loading one half must
rebind only its own parameters while the other half stays on its pinned
CPU master.
"""
from unittest.mock import Mock
import pytest
import torch
import torch.nn as nn
from vllm_omni.diffusion.models.minimax_h3.vae import (
MiniMaxH3VideoVAE,
_VideoVAEPartProxy,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
class _FakeRemoteModel(nn.Module):
def __init__(self):
super().__init__()
self.encoder = nn.Linear(8, 8)
self.quant_conv = nn.Linear(4, 4)
self.post_quant_conv = nn.Linear(4, 4)
self.decoder = nn.Linear(16, 16)
class _FakeRemote(nn.Module):
def __init__(self):
super().__init__()
self.model = _FakeRemoteModel()
def _vae() -> MiniMaxH3VideoVAE:
vae = object.__new__(MiniMaxH3VideoVAE)
nn.Module.__init__(vae) # bare instance: enable Module attribute assignment
vae._device_target = torch.device("cpu")
vae.remote = _FakeRemote()
vae.model = vae.remote.model
vae._stager = None
vae._encoder_stager = None
vae._decoder_stager = None
vae._build_residency_stagers(torch.device("cpu"))
vae.encoder_component = _VideoVAEPartProxy(vae, "encoder")
vae.decoder_component = _VideoVAEPartProxy(vae, "decoder")
return vae
def _ptr(param: torch.nn.Parameter) -> int:
return param.untyped_storage().data_ptr()
def test_split_stagers_snapshot_disjoint_halves():
vae = _vae()
enc_master = _ptr(vae.model.encoder.weight)
dec_master = _ptr(vae.model.decoder.weight)
pqc_master = _ptr(vae.model.post_quant_conv.weight)
assert enc_master != dec_master
vae._load_part_to_device("encoder")
# The encoder half is rebound to freshly allocated staging storage...
assert _ptr(vae.model.encoder.weight) != enc_master
assert _ptr(vae.model.quant_conv.weight) != enc_master
# ...while every decode-half tensor stays on its own CPU master.
assert _ptr(vae.model.decoder.weight) == dec_master
assert _ptr(vae.model.post_quant_conv.weight) == pqc_master
vae._offload_part_to_cpu("encoder")
assert _ptr(vae.model.encoder.weight) == enc_master
def test_part_load_is_idempotent_and_independent():
vae = _vae()
vae._load_part_to_device("decoder")
staged_ptr = _ptr(vae.model.decoder.weight)
vae._load_part_to_device("decoder")
assert _ptr(vae.model.decoder.weight) == staged_ptr
vae._offload_part_to_cpu("decoder")
def test_whole_component_load_and_offload_cover_both_halves():
vae = _vae()
enc_master = _ptr(vae.model.encoder.weight)
dec_master = _ptr(vae.model.decoder.weight)
vae.load_to_device()
assert _ptr(vae.model.encoder.weight) != enc_master
assert _ptr(vae.model.decoder.weight) != dec_master
vae.offload_to_cpu()
assert _ptr(vae.model.encoder.weight) == enc_master
assert _ptr(vae.model.decoder.weight) == dec_master
def test_part_values_survive_a_load_offload_cycle():
vae = _vae()
expected = vae.model.encoder.weight.detach().clone()
vae._load_part_to_device("encoder")
assert torch.equal(vae.model.encoder.weight.detach(), expected)
vae._offload_part_to_cpu("encoder")
assert torch.equal(vae.model.encoder.weight.detach(), expected)
def test_proxy_routes_to_the_matching_half():
vae = _vae()
enc_master = _ptr(vae.model.encoder.weight)
dec_master = _ptr(vae.model.decoder.weight)
vae.encoder_component.load_to_device()
assert _ptr(vae.model.encoder.weight) != enc_master
assert _ptr(vae.model.decoder.weight) == dec_master
vae.encoder_component.offload_to_cpu()
assert _ptr(vae.model.encoder.weight) == enc_master
def test_proxy_does_not_register_the_adapter_as_submodule():
vae = _vae()
proxy = vae.encoder_component
assert proxy._vae is vae
# The back-reference must not recurse through module traversal.
assert all(child is not vae for child in proxy.children())
assert len(list(proxy.parameters())) == 0
def test_component_on_device_unwraps_proxy_for_sequential_offload(monkeypatch):
"""Model-level CPU offload hooks the real video_vae, never the part
proxies: _component_on_device must hand the hooked module to the
sequential context, or _get_sequential_offload_hook raises on the
hook-less proxy on the first encode/decode of every request."""
from contextlib import contextmanager
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as pipeline_module
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
vae = _vae()
seen: list[nn.Module] = []
@contextmanager
def fake_sequential_offload_component(component):
seen.append(component)
yield
monkeypatch.setattr(pipeline_module, "sequential_offload_component", fake_sequential_offload_component)
pipeline = object.__new__(MiniMaxH3Pipeline)
nn.Module.__init__(pipeline)
pipeline._model_cpu_offload_modules = [vae]
with pipeline._component_on_device(vae.encoder_component):
pass
with pipeline._component_on_device(vae.decoder_component):
pass
# A plain component passes through unchanged.
with pipeline._component_on_device(vae):
pass
# Mocks auto-create any attribute, so the unwrap must be a type check:
# a Mock component must reach the sequential context as itself.
mock_vae = Mock()
pipeline._model_cpu_offload_modules = [mock_vae]
with pipeline._component_on_device(mock_vae):
pass
assert seen == [vae, vae, vae, mock_vae]
def test_sequential_offload_target_is_the_hooked_module():
from vllm_omni.diffusion.offloader.sequential_backend import _get_sequential_offload_hook
vae = _vae()
proxy = vae.encoder_component
# The pre-fix failure mode: the proxy carries no hook registry.
with pytest.raises(RuntimeError, match="no sequential offload hook"):
_get_sequential_offload_hook(proxy)
assert proxy.sequential_offload_target is vae
def test_cache_retention_forwarded_to_both_stagers():
vae = _vae()
cache = object()
vae.set_omni_component_cache(cache)
assert vae._encoder_stager.cache_retention is cache
assert vae._decoder_stager.cache_retention is cache
def test_unknown_part_raises():
vae = _vae()
with pytest.raises(ValueError, match="unknown video VAE part"):
vae._part_stager("middle")
def test_missing_checkpoint_structure_falls_back_to_whole_stager():
vae = object.__new__(MiniMaxH3VideoVAE)
nn.Module.__init__(vae)
vae._device_target = torch.device("cpu")
class BareRemote(nn.Module):
def __init__(self):
super().__init__()
self.model = nn.Linear(4, 4) # no encoder/decoder submodules
vae.remote = BareRemote()
vae.model = vae.remote.model
vae._stager = None
vae._encoder_stager = None
vae._decoder_stager = None
vae._build_residency_stagers(torch.device("cpu"))
assert vae._stager is not None
assert vae._encoder_stager is None
assert vae._decoder_stager is None
# Part loads fall back to the whole-module stager.
vae._load_part_to_device("decoder")
assert vae._stager.loaded
vae._offload_part_to_cpu("decoder")
assert not vae._stager.loaded
@@ -0,0 +1,335 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU tests for the temporal VAE forks and the uint8 output transfer.
The encode fork must feed bit-identical frames to the checkpoint's chunk
loop (padding moves from a whole-video cat into the tail chunk), the decode
fork must produce the same video as the legacy fp32-buffer chain
(revert -> clamp -> *255 -> round -> uint8), and the output transfer must
match the previous ``.to(uint8, contiguous)`` conversion.
"""
from typing import Any, cast
import pytest
import torch
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
_prepare_minimax_h3_video_output,
)
from vllm_omni.diffusion.models.minimax_h3.vae_temporal import (
_decode_temporal_streaming_uint8,
_encode_temporal_tail_pad,
install_temporal_stream_patches,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
CLIP = 4
# ---------------------------------------------------------------------------
# encode_temporal fork
# ---------------------------------------------------------------------------
class _EncodeModel:
"""Minimal stand-in implementing the checkpoint's encode_temporal."""
# Attached per test case as capability probes for the install checks;
# annotations only, the fork must keep working when they are absent.
encode_temporal: Any
_decode_temporal_streaming: Any
processor: Any
def __init__(self, *, isolated_first=False, isolated_last=False, key_frame=False, frame_drop=0):
self.clip_length = CLIP
self.isolated_first_frame = isolated_first
self.frame_pre_padding = 0
self.isolated_key_frame = key_frame
self.token_drop = 0
self.isolated_last_frame = isolated_last
self.frame_drop = frame_drop
self.encoded = []
def _adaptive_encode(self, clip):
self.encoded.append(clip.clone())
return clip # identity: z_list entries are the clips themselves
def reference_encode_temporal(self, x):
"""The checkpoint's original: whole-video pad, then chunk."""
offset_frame = 1 if self.isolated_first_frame and self.frame_pre_padding == 0 else 0
if x.shape[2] % self.clip_length != offset_frame:
pad_size = (offset_frame - x.shape[2]) % self.clip_length
pad_frames = x[:, :, -1:].repeat(1, 1, pad_size, 1, 1)
x = torch.cat([x, pad_frames], dim=2)
num_chunks = (x.shape[2] - offset_frame) // self.clip_length
z_list = []
for i in range(num_chunks):
start_idx = i * self.clip_length + offset_frame
end_idx = (i + 1) * self.clip_length + offset_frame
clip_x = x[:, :, start_idx:end_idx, :, :]
if self.isolated_key_frame:
key_frame = clip_x[:, :, :1, :, :]
z_key = self._adaptive_encode(key_frame)
if clip_x.shape[2] > 1:
z_video = self._adaptive_encode(clip_x[:, :, 1:, :, :])
z = torch.cat([z_key, z_video], dim=2)
else:
z = z_key
else:
z = self._adaptive_encode(clip_x)
z_list.append(z)
z = torch.cat(z_list, dim=2)
if self.token_drop > 0:
z = z[:, :, : -self.token_drop]
if self.isolated_first_frame:
z_first = self._adaptive_encode(x[:, :, :1, :, :])
z = torch.cat([z_first, z], dim=2)
if self.isolated_last_frame:
frame_num = x.shape[2]
last_idx = frame_num - self.frame_drop + offset_frame
z_last = self._adaptive_encode(x[:, :, last_idx : last_idx + 1, :, :])
z = torch.cat([z, z_last], dim=2)
return z
def _video_x(t, c=3, h=4, w=4):
return torch.arange(t * c * h * w, dtype=torch.float32).reshape(1, c, t, h, w) / 255.0
@pytest.mark.parametrize("num_frames", [8, 10, 11, 13])
@pytest.mark.parametrize(
"flags",
[
{},
{"isolated_first": True},
{"isolated_last": True},
{"isolated_first": True, "isolated_last": True},
{"key_frame": True},
],
)
def test_encode_fork_matches_whole_video_pad_bitwise(num_frames, flags):
x = _video_x(num_frames)
ref_model = _EncodeModel(**flags)
expected = ref_model.reference_encode_temporal(x.clone())
# Empty-slice encodes (out-of-range isolated-last indexes are no-ops in
# the checkpoint) carry no content; compare only real calls.
ref_calls = [c.clone() for c in ref_model.encoded if c.shape[2] > 0]
fork_model = _EncodeModel(**flags)
out = _encode_temporal_tail_pad(fork_model, x.clone())
assert out.shape == expected.shape
assert torch.equal(out, expected)
assert len(fork_model.encoded) == len(ref_calls)
for got, want in zip(fork_model.encoded, ref_calls):
assert torch.equal(got, want)
def test_encode_fork_avoids_whole_video_pad_for_misaligned_input():
# 375 frames with an asymmetric config (isolated last frame only):
# the checkpoint always pads (375 % 17 != 0); the fork must encode the
# tail chunk with an in-chunk pad instead of concatenating the video.
model = _EncodeModel(isolated_last=True, frame_drop=16)
model.clip_length = 17
x = _video_x(375, c=1, h=2, w=2)
_encode_temporal_tail_pad(model, x)
# 375 % 17 == 1 → pad 16 inside the tail chunk; the isolated last frame
# resolves to the final real frame (pad frames replicate it).
assert len(model.encoded) == (375 // 17) + 1 + 1
assert torch.equal(model.encoded[-1].flatten(), x[:, :, -1:, :, :].flatten())
# ---------------------------------------------------------------------------
# _decode_temporal_streaming uint8 fork
# ---------------------------------------------------------------------------
DENORM_MEAN = (-2.11628412, -2.03571429, -1.80444444)
DENORM_STD = (4.36681223, 4.46428571, 4.44444444)
class _FakeDenorm:
def __init__(self):
self.mean = DENORM_MEAN
self.std = DENORM_STD
class _FakeProcessor:
def __init__(self):
self.transform_rev = _FakeDenorm()
class _DecodeModel:
"""Minimal stand-in for the checkpoint's streaming decode contract."""
def __init__(self, *, token_drop=0, frame_overlap=0):
self.tokens_chunk_size = 2
self.vae_ratio_t = 1
self.token_overlap = 1 if token_drop else 0
self.token_drop = token_drop
self.frame_pre_padding = 0
self.frame_overlap = frame_overlap
self.processor = _FakeProcessor()
def _adaptive_decode(self, clip_z):
# Deterministic pseudo-decode: map latent tokens to frames 1:1.
return torch.sin(clip_z * 3.0)
def blend(self, a, b, overlap, dim=-3):
return (a + b) / 2.0
def _decode_temporal_pad_frames(self, z, pad_tokens):
return int(pad_tokens)
def _decode_temporal_output_frame_plan(self, z, z_head, z_tail, num_chunks, pad_tokens):
total = num_chunks * self.tokens_chunk_size * self.vae_ratio_t
if z_head is not None:
total += 1
if z_tail is not None:
total += 1
if self.token_drop > 0:
total += self.frame_overlap
return int(total), int(pad_tokens), int(total - pad_tokens)
def legacy_streaming(self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype):
"""The checkpoint's original: accumulate in the cat dtype, quantize later."""
total_frames, _, output_frames = self._decode_temporal_output_frame_plan(
z, z_head, z_tail, num_chunks, pad_tokens
)
chunk_dec = self.tokens_chunk_size * self.vae_ratio_t
split_count = int(self.token_drop > 0) + 1
dec = None
dec_overlap = None
write_pos = 0
def write_part(part):
nonlocal dec, write_pos
if dec is None:
out_shape = list(part.shape)
out_shape[2] = output_frames
dec = torch.empty(out_shape, dtype=part.dtype)
n = min(part.shape[2], dec.shape[2] - write_pos)
if n > 0:
dec[:, :, write_pos : write_pos + n].copy_(part[:, :, :n])
write_pos += n
for i in range(num_chunks):
t_start = i * self.tokens_chunk_size
t_end = t_start + self.tokens_chunk_size + self.token_overlap
clip_z = z[:, :, t_start:t_end]
clip_dec = self._adaptive_decode(clip_z)
if temporal_cat_dtype is not None and clip_dec.dtype != temporal_cat_dtype:
clip_dec = clip_dec.to(temporal_cat_dtype)
for j in range(split_count):
chunk = clip_dec[:, :, j * chunk_dec : min(j * chunk_dec + chunk_dec, clip_dec.shape[2])]
if j == 0:
if dec_overlap is not None:
chunk = self.blend(dec_overlap, chunk, self.frame_overlap, dim=-3)
dec_overlap = None
write_part(chunk)
else:
dec_overlap = chunk.contiguous()
if i == num_chunks - 1 and dec_overlap is not None:
write_part(dec_overlap)
dec_overlap = None
# Legacy post-decode chain: revert in place in the decoded dtype, then
# the adapter's ``frames.float()`` before the pipeline's quantizer.
assert dec is not None
mean = torch.tensor(DENORM_MEAN, dtype=dec.dtype).view(1, 3, 1, 1, 1)
std = torch.tensor(DENORM_STD, dtype=dec.dtype).view(1, 3, 1, 1, 1)
dec.sub_(mean).div_(std).clamp_(0.0, 1.0)
dec = dec.float()
dec.mul_(255.0).round_()
return dec.to(torch.uint8)
@pytest.mark.parametrize("token_drop,frame_overlap", [(0, 0), (1, 1)])
@pytest.mark.parametrize("temporal_cat_dtype", [None, torch.float16, torch.bfloat16])
def test_decode_fork_matches_legacy_chain_bitwise(token_drop, frame_overlap, temporal_cat_dtype):
# The quantizer must see FP32 no matter the cat dtype: the legacy adapter
# returned frames.float() before the pipeline multiplied and rounded, so
# an fp16/bf16 cat dtype only narrows denormalization, never the rounding.
torch.manual_seed(0)
tokens = 2 * 2 + (1 if token_drop else 0) # num_chunks * tokens_chunk_size
z = torch.randn(1, 3, tokens, 4, 4)
legacy = _DecodeModel(token_drop=token_drop, frame_overlap=frame_overlap)
expected = legacy.legacy_streaming(z.clone(), None, None, 2, 0, temporal_cat_dtype)
fork = _DecodeModel(token_drop=token_drop, frame_overlap=frame_overlap)
out = _decode_temporal_streaming_uint8(fork, z.clone(), None, None, 2, 0, temporal_cat_dtype)
assert out.dtype == torch.uint8
assert out.shape == expected.shape
assert torch.equal(out, expected)
def test_decode_fork_rejects_empty_plan():
model = _DecodeModel()
model.tokens_chunk_size = 0
with pytest.raises(ValueError, match="non-positive output_frames"):
_decode_temporal_streaming_uint8(model, torch.randn(1, 3, 0, 4, 4), None, None, 0, 0, None)
# ---------------------------------------------------------------------------
# install + escape hatch
# ---------------------------------------------------------------------------
def test_install_binds_both_forks():
model = _EncodeModel()
# cast(Any, ...) keeps mypy from narrowing the probes to the lambda type,
# which has no __func__ for the bound-method assertions below.
model.encode_temporal = cast(Any, lambda x: x) # placeholder for the capability check
model._decode_temporal_streaming = cast(Any, lambda *a: None)
model.processor = _FakeProcessor()
install_temporal_stream_patches(model)
assert model.encode_temporal.__func__ is _encode_temporal_tail_pad
assert model._decode_temporal_streaming.__func__ is _decode_temporal_streaming_uint8
def test_install_skips_decode_fork_without_denorm_contract():
model = _EncodeModel()
model.encode_temporal = cast(Any, lambda x: x)
model._decode_temporal_streaming = cast(Any, lambda *a: None)
install_temporal_stream_patches(model) # no processor -> keep original
assert model.encode_temporal.__func__ is _encode_temporal_tail_pad
assert not hasattr(model._decode_temporal_streaming, "__func__")
def test_install_respects_legacy_env(monkeypatch):
monkeypatch.setenv("VLLM_OMNI_VAE_LEGACY_TEMPORAL", "1")
model = _EncodeModel()
install_temporal_stream_patches(model)
assert not hasattr(getattr(model, "encode_temporal", None), "__func__")
# ---------------------------------------------------------------------------
# _prepare_minimax_h3_video_output
# ---------------------------------------------------------------------------
def test_prepare_output_matches_previous_conversion():
video = torch.randn(1, 3, 4, 6, 8).clamp_(-1, 2)
legacy = (
video.detach()
.float()
.clamp(0, 1)
.mul(255)
.round()
.permute(0, 2, 3, 4, 1)
.to(dtype=torch.uint8, memory_format=torch.contiguous_format)
)
out = _prepare_minimax_h3_video_output(video.clone())
assert out.dtype == torch.uint8
assert out.shape == (1, 4, 6, 8, 3)
assert out.is_contiguous()
assert torch.equal(out, legacy)
def test_prepare_output_uint8_passthrough():
video = torch.randint(0, 256, (1, 3, 4, 6, 8), dtype=torch.uint8)
out = _prepare_minimax_h3_video_output(video)
assert out.dtype == torch.uint8
assert out.is_contiguous()
assert torch.equal(out, video.permute(0, 2, 3, 4, 1).contiguous())
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
@@ -0,0 +1,500 @@
#!/usr/bin/env python
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""π0.5 parity against LeRobot.
Reference: **lerobot 0.6.1**, transformers 5.5.4, torch 2.11.0+cpu. The oracle
is LeRobot's own code, so a later version changing its π0.5 implementation
fails this without anything here regressing — diff ``policies/pi05/`` first.
Compares final action chunks under fixed noise: float32 exactly, bfloat16
against LeRobot's own bfloat16 layout within ``BF16_ATOL``.
Needs a separate ``lerobot[pi]`` venv (its dependencies conflict with
vllm-omni's) with the vllm-omni ``pi05`` package importable::
python -m pytest tests/diffusion/models/pi05/test_pi05_parity.py -v -s
Skips itself when LeRobot is absent. ``PI05_PARITY_MODEL_PATH`` points at a
local checkpoint to skip the HF download.
"""
from __future__ import annotations
import copy
import importlib.util
import os
import pytest
import torch
# local_model: needs real weights + a lerobot venv, so it runs locally rather
# than in the ready-CI. Additionally gated on lerobot being importable.
_HAS_LEROBOT = importlib.util.find_spec("lerobot") is not None
# cpu: the comparison runs on CPU in float32 so both implementations see
# identical numerics; local_model: it needs a lerobot venv that CI does not have.
pytestmark = [pytest.mark.local_model, pytest.mark.diffusion, pytest.mark.cpu]
# ─── Config (fixed; matches LeRobot defaults for a π0.5 checkpoint) ────
DEVICE = "cpu"
DTYPE_STR = "float32"
ATOL = 1e-4
BF16_ATOL = 5e-2 # measured 2.95e-2, reproducible; margin is for version drift
NUM_STEPS = 10
BATCH_SIZE = 2
ACTION_DIM = 32
STATE_DIM = 32
ACTION_HORIZON = 50
MAX_TOKEN_LEN = 200 # π0 uses 48
NUM_STATE_BINS = 256
MODEL_PATH = os.environ.get("PI05_PARITY_MODEL_PATH", "lerobot/pi05_base")
CAMERAS = ("base_0_rgb", "left_wrist_0_rgb", "right_wrist_0_rgb")
def _resolve_checkpoint_dir() -> str:
if os.path.isdir(MODEL_PATH):
return MODEL_PATH
from huggingface_hub import snapshot_download
return snapshot_download(repo_id=MODEL_PATH, repo_type="model")
# ─── Dummy dataset stats ──────────────────────────────────────────────
def _dummy_dataset_stats() -> dict:
"""Identity-ish stats. π0.5 normalizes with QUANTILES by default, so q01/q99
carry the load here where π0's mean/std would.
Using dummy stats means this test needs no checkpoint with real
``norm_stats`` — it verifies numerical equivalence, not calibration.
"""
return {
"observation.state": {
"mean": torch.zeros(STATE_DIM),
"std": torch.ones(STATE_DIM),
"q01": -torch.ones(STATE_DIM),
"q99": torch.ones(STATE_DIM),
},
"action": {
"mean": torch.zeros(ACTION_DIM),
"std": torch.ones(ACTION_DIM),
"q01": -torch.ones(ACTION_DIM),
"q99": torch.ones(ACTION_DIM),
},
"images": {
cam: {
"mean": torch.zeros(3, 224, 224),
"std": torch.ones(3, 224, 224),
"q01": torch.zeros(3, 224, 224),
"q99": torch.ones(3, 224, 224),
}
for cam in CAMERAS
},
}
def _create_dummy_batch(batch_size: int = BATCH_SIZE, num_views: int = 3, device: str = DEVICE) -> dict:
"""Reproducible dummy inputs — identical across both implementations."""
g = torch.Generator(device="cpu").manual_seed(0)
prompt = "Pick up the red block and place it in the bin"
batch = {
"observation.state": torch.randn(batch_size, STATE_DIM, generator=g, dtype=torch.float32).to(device),
"action": torch.randn(batch_size, ACTION_HORIZON, ACTION_DIM, generator=g, dtype=torch.float32).to(device),
"task": [prompt for _ in range(batch_size)],
}
for cam in CAMERAS[:num_views]:
batch[f"observation.images.{cam}"] = torch.rand(batch_size, 3, 224, 224, generator=g, dtype=torch.float32).to(
device
)
return batch
# ─── Instantiation ────────────────────────────────────────────────────
def _instantiate_lerobot(use_relative_actions: bool = False, action_feature_names=None, dtype: str = DTYPE_STR):
from lerobot.policies.pi05 import PI05Policy
from lerobot.policies.pi05.processor_pi05 import make_pi05_pre_post_processors
policy = PI05Policy.from_pretrained(MODEL_PATH, strict=True, dtype=dtype)
policy.to(DEVICE)
policy.config.device = DEVICE
policy.config.use_relative_actions = use_relative_actions
if action_feature_names is not None:
# Must land on the config *before* the processors are built:
# make_pi05_pre_post_processors captures action_feature_names into
# RelativeActionsProcessorStep at construction time, and its _build_mask
# treats ``action_names is None`` as "shift every dim" — including the
# gripper that relative_exclude_joints is supposed to hold back. Setting
# it afterwards leaves the step with the all-True mask and silently
# compares two different transforms.
policy.config.action_feature_names = action_feature_names
policy.eval()
pre, post = make_pi05_pre_post_processors(config=policy.config, dataset_stats=_dummy_dataset_stats())
return policy, pre, post
def _instantiate_vllm_omni(
use_relative_actions: bool = False,
action_feature_names=None,
dtype: str = DTYPE_STR,
):
"""Build the vllm-omni π0.5 model in isolation (no pipeline, no engine)."""
from vllm_omni.diffusion.models.pi05 import Pi05Config, Pi05ForActionPrediction
from vllm_omni.diffusion.models.pi05.pipeline_pi05 import _set_inference_dtype
cfg = Pi05Config(
max_action_dim=ACTION_DIM,
max_state_dim=STATE_DIM,
chunk_size=ACTION_HORIZON,
num_inference_steps=NUM_STEPS,
tokenizer_max_length=MAX_TOKEN_LEN,
state_num_bins=NUM_STATE_BINS,
dtype=dtype,
use_relative_actions=use_relative_actions,
relative_exclude_joints=["gripper"] if action_feature_names else [],
action_feature_names=action_feature_names,
)
model = Pi05ForActionPrediction(cfg)
_set_inference_dtype(model, getattr(torch, dtype))
model.to(device=DEVICE)
_load_lerobot_weights(model)
model.eval()
return model, cfg
def _load_lerobot_weights(model):
import safetensors.torch
path = os.path.join(_resolve_checkpoint_dir(), "model.safetensors")
state = safetensors.torch.load_file(path)
filled = model.load_weights(list(state.items()))
assert filled, "no weights were loaded — the remap rules are broken"
def _extract_lerobot_model_inputs(lerobot_policy, processed_batch):
"""Mimic what ``PI05Policy.predict_action_chunk`` feeds into
``self.model.sample_actions``. Using these *exact* tensors for vllm-omni
means any divergence must come from the core model, not preprocessing.
Note π0.5 has no ``prepare_state`` step: the state is already inside
``lang_tokens``.
"""
from lerobot.utils.constants import (
OBS_LANGUAGE_ATTENTION_MASK,
OBS_LANGUAGE_TOKENS,
)
images, img_masks = lerobot_policy._preprocess_images(processed_batch)
return images, img_masks, processed_batch[OBS_LANGUAGE_TOKENS], processed_batch[OBS_LANGUAGE_ATTENTION_MASK]
def _make_fixed_noise(batch_size: int, device: str) -> torch.Tensor:
g = torch.Generator(device="cpu").manual_seed(42)
return torch.randn(batch_size, ACTION_HORIZON, ACTION_DIM, generator=g, dtype=torch.float32).to(device)
# ─── Main parity test ─────────────────────────────────────────────────
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
@pytest.mark.parametrize("num_views", [1, 2, 3])
def test_pi05_vllm_omni_vs_lerobot(num_views):
lerobot_policy, lerobot_pre, _ = _instantiate_lerobot()
omni_model, _ = _instantiate_vllm_omni()
raw_batch = _create_dummy_batch(num_views=num_views)
processed = lerobot_pre(copy.deepcopy(raw_batch))
images, img_masks, lang_tokens, lang_masks = _extract_lerobot_model_inputs(lerobot_policy, processed)
noise = _make_fixed_noise(raw_batch["observation.state"].shape[0], DEVICE)
# The input contract: 256 tokens per view + a constant 200 text tokens.
assert lang_tokens.shape[1] == MAX_TOKEN_LEN, "π0.5 must pad text to exactly 200 tokens"
with torch.no_grad():
lerobot_actions = lerobot_policy.model.sample_actions(
images, img_masks, lang_tokens, lang_masks, noise=noise, num_steps=NUM_STEPS
)
omni_actions = omni_model.sample_actions(
images=images,
image_masks=img_masks,
lang_tokens=lang_tokens,
lang_masks=lang_masks,
noise=noise,
num_steps=NUM_STEPS,
)
diff = (lerobot_actions.float() - omni_actions.float()).abs()
print(f"[parity] views={num_views} |Δ| max={diff.max().item():.2e} mean={diff.mean().item():.2e}")
if not torch.allclose(lerobot_actions.float(), omni_actions.float(), atol=ATOL):
_diagnose_divergence(lerobot_policy.model, omni_model, images, img_masks, lang_tokens, lang_masks, noise)
assert torch.allclose(lerobot_actions.float(), omni_actions.float(), atol=ATOL), (
f"actions differ beyond atol={ATOL}; max_diff={diff.max().item():.2e}"
)
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
def test_pi05_bfloat16_matches_lerobot_bfloat16():
"""Both sides in bfloat16, so this compares implementations rather than
measuring bfloat16's drift from float32."""
# LeRobot applies the layout in PaliGemmaWithExpertModel.__init__ from
# config.dtype, so the policy must be built as bfloat16 rather than cast after.
lerobot_policy, lerobot_pre, _ = _instantiate_lerobot(dtype="bfloat16")
omni_model, _ = _instantiate_vllm_omni(dtype="bfloat16")
raw_batch = _create_dummy_batch(num_views=3)
processed = lerobot_pre(copy.deepcopy(raw_batch))
images, img_masks, lang_tokens, lang_masks = _extract_lerobot_model_inputs(lerobot_policy, processed)
noise = _make_fixed_noise(raw_batch["observation.state"].shape[0], DEVICE)
with torch.no_grad():
lerobot_actions = lerobot_policy.model.sample_actions(
images, img_masks, lang_tokens, lang_masks, noise=noise, num_steps=NUM_STEPS
)
omni_actions = omni_model.sample_actions(
images=images,
image_masks=img_masks,
lang_tokens=lang_tokens,
lang_masks=lang_masks,
noise=noise,
num_steps=NUM_STEPS,
)
diff = (lerobot_actions.float() - omni_actions.float()).abs()
print(f"[parity] bfloat16 |Δ| max={diff.max().item():.2e} mean={diff.mean().item():.2e}")
assert torch.allclose(lerobot_actions.float(), omni_actions.float(), atol=BF16_ATOL), (
f"bfloat16 actions differ beyond atol={BF16_ATOL}; max_diff={diff.max().item():.2e}"
)
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
def test_pi05_prompt_parity():
"""π0.5's state is carried as prompt tokens, so the tokenized prompt is
itself part of the contract. A bin-index mismatch here (e.g. from running
the discretizer before the normalizer) would otherwise only surface as a
small, plausible-looking action difference.
"""
from transformers import AutoTokenizer
from vllm_omni.diffusion.models.pi05.processor_pi05 import (
apply_norm,
build_norm_stats,
build_pi05_prompt,
tokenize_prompt,
)
lerobot_policy, lerobot_pre, _ = _instantiate_lerobot()
raw_batch = _create_dummy_batch(batch_size=1)
processed = lerobot_pre(copy.deepcopy(raw_batch))
from lerobot.utils.constants import OBS_LANGUAGE_TOKENS
lerobot_tokens = processed[OBS_LANGUAGE_TOKENS][0]
stats = _dummy_dataset_stats()["observation.state"]
norm = build_norm_stats({"state": {"q01": stats["q01"], "q99": stats["q99"]}}, "state")
normalized = apply_norm(raw_batch["observation.state"][0].float(), norm)
prompt = build_pi05_prompt(
task=raw_batch["task"][0],
normalized_state=normalized.numpy(),
state_num_bins=NUM_STATE_BINS,
)
tokenizer = AutoTokenizer.from_pretrained("google/paligemma-3b-pt-224", padding_side="right")
ids, _ = tokenize_prompt(tokenizer, prompt, MAX_TOKEN_LEN)
assert len(ids) == MAX_TOKEN_LEN
assert torch.equal(torch.tensor(ids), lerobot_tokens.cpu()), "π0.5 prompt tokens diverge from LeRobot"
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
@pytest.mark.parametrize("state_dim", [7, STATE_DIM])
def test_pi05_prompt_parity_across_state_widths(state_dim):
"""``pi05_base`` declares a 32-dim state, which is exactly ``max_state_dim``,
so the full-policy parity above cannot tell padding apart from the real
width. This drives LeRobot's tokenizer step directly, at both widths.
"""
from lerobot.lerobot_types import TransitionKey
from lerobot.policies.pi05.processor_pi05 import Pi05PrepareStateTokenizerProcessorStep
from lerobot.utils.constants import OBS_STATE
from vllm_omni.diffusion.models.pi05.processor_pi05 import build_pi05_prompt
g = torch.Generator().manual_seed(state_dim)
state = torch.randn(state_dim, generator=g, dtype=torch.float32)
task = "pick up the red block"
step = Pi05PrepareStateTokenizerProcessorStep(max_state_dim=STATE_DIM)
transition = step(
{
TransitionKey.OBSERVATION: {OBS_STATE: state.unsqueeze(0)},
TransitionKey.COMPLEMENTARY_DATA: {"task": [task]},
}
)
lerobot_prompt = transition[TransitionKey.COMPLEMENTARY_DATA]["task"][0]
prompt = build_pi05_prompt(task=task, normalized_state=state.numpy())
assert prompt == lerobot_prompt
assert len(prompt.split("State: ")[1].split(";")[0].split()) == state_dim
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
@pytest.mark.parametrize(
"lerobot_mode,stats,ours,state,state_dim",
[
("MEAN_STD", {"mean": [0.0], "std": [1e-7]}, {"mode": "mean_std", "mean": [0.0], "std": [1e-7]}, [1e-7], 1),
("QUANTILES", {"q01": [0.0], "q99": [1e-7]}, {"mode": "quantile", "q01": [0.0], "q99": [1e-7]}, [1e-7], 1),
("QUANTILES", {"q01": [2.0], "q99": [2.0]}, {"mode": "quantile", "q01": [2.0], "q99": [2.0]}, [2.0], 1),
(
"QUANTILES",
{"q01": [0.0] * 4, "q99": [10.0] * 4},
{"mode": "quantile", "q01": [0.0] * 4, "q99": [10.0] * 4},
[-10.0, 0.0, 5.0, 20.0],
4,
),
(
"MEAN_STD",
{"mean": [1.0, 2.0], "std": [0.5, 4.0]},
{"mode": "mean_std", "mean": [1.0, 2.0], "std": [0.5, 4.0]},
[3.0, -6.0],
2,
),
("MIN_MAX", {"min": [-1.0], "max": [3.0]}, {"mode": "min_max", "min": [-1.0], "max": [3.0]}, [2.0], 1),
],
)
def test_pi05_state_normalization_parity(lerobot_mode, stats, ours, state, state_dim):
"""State normalization against LeRobot's ``NormalizerProcessorStep``.
The full-policy parity test feeds both sides stats with a wide range, so it
cannot see the denominator rule. A wrongly normalized state still discretizes
into a well-formed prompt: for ``q01=0, q99=1e-7`` a threshold substitution
moves the state token from bin 255 to bin 0.
"""
from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
from lerobot.processor.normalize_processor import NormalizerProcessorStep
from lerobot.utils.constants import OBS_STATE
from vllm_omni.diffusion.models.pi05.processor_pi05 import apply_norm, build_norm_stats
step = NormalizerProcessorStep(
features={OBS_STATE: PolicyFeature(type=FeatureType.STATE, shape=(state_dim,))},
norm_map={FeatureType.STATE: getattr(NormalizationMode, lerobot_mode)},
stats={OBS_STATE: {k: torch.tensor(v, dtype=torch.float32) for k, v in stats.items()}},
)
expected = step._apply_transform(
torch.tensor(state, dtype=torch.float32), OBS_STATE, FeatureType.STATE, inverse=False
)
actual = apply_norm(torch.tensor(state, dtype=torch.float32), build_norm_stats({"state": ours}, "state"))
assert torch.allclose(actual, expected, atol=1e-6)
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
def test_pi05_relative_actions_parity():
"""With ``use_relative_actions=True`` the checkpoint's norm_stats live in
relative space, so the post-pipeline must add the raw state back. Compares
the full pre → model → post path against LeRobot's."""
from lerobot.utils.constants import (
OBS_LANGUAGE_ATTENTION_MASK,
OBS_LANGUAGE_TOKENS,
)
from vllm_omni.diffusion.models.pi05.processor_pi05 import (
Pi05RelativeActions,
apply_norm,
build_norm_stats,
)
action_names = [f"joint_{i}" for i in range(ACTION_DIM - 1)] + ["gripper"]
lerobot_policy, lerobot_pre, lerobot_post = _instantiate_lerobot(
use_relative_actions=True, action_feature_names=action_names
)
omni_model, cfg = _instantiate_vllm_omni(use_relative_actions=True, action_feature_names=action_names)
raw_batch = _create_dummy_batch()
processed = lerobot_pre(copy.deepcopy(raw_batch))
images, img_masks = lerobot_policy._preprocess_images(processed)
lang_tokens = processed[OBS_LANGUAGE_TOKENS]
lang_masks = processed[OBS_LANGUAGE_ATTENTION_MASK]
noise = _make_fixed_noise(raw_batch["observation.state"].shape[0], DEVICE)
with torch.no_grad():
lerobot_actions = lerobot_post(
lerobot_policy.model.sample_actions(
images, img_masks, lang_tokens, lang_masks, noise=noise, num_steps=NUM_STEPS
)
)
omni_raw = omni_model.sample_actions(
images=images,
image_masks=img_masks,
lang_tokens=lang_tokens,
lang_masks=lang_masks,
noise=noise,
num_steps=NUM_STEPS,
)
rel = Pi05RelativeActions(
enabled=True,
exclude_joints=cfg.relative_exclude_joints,
action_names=action_names,
max_action_dim=ACTION_DIM,
)
omni_actions = apply_norm(omni_raw, build_norm_stats(cfg.norm_stats, "action"), inverse=True)
# Per-sample reference state, matching LeRobot: its
# RelativeActionsProcessorStep caches the whole (B, D) state on the input
# side and shifts each sample by its own row. Passing a single row here
# would shift every sample by sample 0's state, which only agrees for B=1.
omni_actions = rel.to_absolute(omni_actions, raw_batch["observation.state"])
lerobot_actions = torch.as_tensor(lerobot_actions).float()
assert torch.allclose(lerobot_actions, omni_actions.float(), atol=ATOL), (
"relative-action path diverges from LeRobot; "
f"max_diff={(lerobot_actions - omni_actions.float()).abs().max().item():.2e}"
)
# ─── Per-stage divergence diagnostics ─────────────────────────────────
@torch.no_grad()
def _diagnose_divergence(lerobot_flow_model, omni_model, images, img_masks, lang_tokens, lang_masks, noise):
"""Localize a numerical mismatch to a specific stage:
1. Prefix embeddings (SigLIP / embed_tokens / projector) — image vs lang.
2. Prefix KV cache layer 0 — PaliGemma LM attention.
3. The AdaRMS timestep conditioning vector.
4. A single denoise_step velocity at t=1.0 — action expert forward.
"""
from vllm_omni.diffusion.models.pi05.modeling_pi05 import (
make_att_2d_masks,
prepare_attention_masks_4d,
)
lr_embs, lr_pad, lr_att = lerobot_flow_model.embed_prefix(images, img_masks, lang_tokens, lang_masks)
sg_embs, sg_pad, sg_att = omni_model.embed_prefix(images, img_masks, lang_tokens, lang_masks)
print(f"[diag] prefix_embs max|Δ| = {(lr_embs.float() - sg_embs.float()).abs().max().item():.2e}")
print(f"[diag] prefix_pad_masks equal: {torch.equal(lr_pad, sg_pad)}")
img_len = 256 * len(images)
print(f"[diag] image slice max|Δ| = {(lr_embs[:, :img_len] - sg_embs[:, :img_len]).abs().max().item():.2e}")
print(f"[diag] lang slice max|Δ| = {(lr_embs[:, img_len:] - sg_embs[:, img_len:]).abs().max().item():.2e}")
prefix_att_4d = prepare_attention_masks_4d(make_att_2d_masks(sg_pad, sg_att))
prefix_pos = torch.cumsum(sg_pad, dim=1) - 1
_, sg_kv = omni_model.paligemma_with_expert.forward(
attention_mask=prefix_att_4d,
position_ids=prefix_pos,
past_key_values=None,
inputs_embeds=[sg_embs, None],
use_cache=True,
)
t = torch.ones(noise.shape[0], dtype=torch.float32)
sg_cond = omni_model.embed_timestep(t)
print(f"[diag] adarms cond: shape={tuple(sg_cond.shape)} mean={sg_cond.mean().item():+.4f}")
sg_vt = omni_model.denoise_step(sg_pad, sg_kv, noise, t)
print(f"[diag] denoise_step(t=1) v_t mean={sg_vt.mean().item():+.6f}")
if __name__ == "__main__":
test_pi05_vllm_omni_vs_lerobot(3)
Loaded 100 of 769 files, more files were not shown because too many files have changed in this diff. Show more