Merge main and preserve dispatch backlog across monitor windows
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
commit
ea7de7e3de
769 files changed
+73802
-22805
No files matched your search
@@ -3,6 +3,7 @@
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from vllm_omni.attention import fish_kvcache_attn, fish_kvcache_backend
|
||||
|
||||
@@ -113,6 +114,15 @@ def test_fish_kvcache_enabled_by_default(monkeypatch):
|
||||
assert not fish_kvcache_attn.is_fish_kvcache_attn_required()
|
||||
|
||||
|
||||
def test_fish_kvcache_fast_path_is_unavailable_off_cuda(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
fish_kvcache_attn, "_triton_backend", lambda: pytest.fail("Triton backend must not load off CUDA")
|
||||
)
|
||||
|
||||
assert not fish_kvcache_attn.is_available()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", "1", "true", "yes", "on", "required"])
|
||||
def test_fish_kvcache_enabled_values_are_consistent(monkeypatch, value):
|
||||
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", value)
|
||||
|
||||
@@ -1,15 +1,289 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Tests for omni_duplex_eval_media — PyAV-backed media helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
import av
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.benchmarks.duplex.omni_duplex_eval_media import materialize_media
|
||||
from vllm_omni.benchmarks.duplex.omni_duplex_eval_media import (
|
||||
_ffmpeg_q_to_pil_quality,
|
||||
extract_jpeg,
|
||||
materialize_media,
|
||||
read_audio_pcm16,
|
||||
video_duration,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers: synthetic test media generated with PyAV (no external files)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_synthetic_mp4(duration: float = 1.0, fps: int = 30) -> bytes:
|
||||
"""Return an MP4 byte blob with a solid-color pattern.
|
||||
|
||||
Uses the built-in ``mpeg4`` encoder — available everywhere PyAV's bundled
|
||||
FFmpeg is present — so no external codec dependency.
|
||||
"""
|
||||
buf = io.BytesIO()
|
||||
total_frames = int(duration * fps)
|
||||
with av.open(buf, mode="w", format="mp4") as container:
|
||||
stream = container.add_stream("mpeg4", rate=fps)
|
||||
stream.width = 64
|
||||
stream.height = 64
|
||||
stream.pix_fmt = "yuv420p"
|
||||
for i in range(total_frames):
|
||||
frame = av.VideoFrame.from_ndarray(np.full((64, 64, 3), (i * 8) % 256, dtype=np.uint8), format="rgb24")
|
||||
for pkt in stream.encode(frame):
|
||||
container.mux(pkt)
|
||||
for pkt in stream.encode():
|
||||
container.mux(pkt)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _make_synthetic_wav(duration: float = 1.0, sample_rate: int = 16000) -> bytes:
|
||||
"""Return a 16-bit mono PCM WAV byte blob (440 Hz sine wave)."""
|
||||
import wave as _wave
|
||||
|
||||
n = int(sample_rate * duration)
|
||||
samples = (np.sin(2 * np.pi * 440 * np.arange(n) / sample_rate) * 32767).astype(np.int16)
|
||||
buf = io.BytesIO()
|
||||
with _wave.open(buf, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(sample_rate)
|
||||
wf.writeframes(samples.tobytes())
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _make_synthetic_ogg(duration: float = 1.0, sample_rate: int = 16000) -> bytes:
|
||||
"""Return an OGG Vorbis byte blob (440 Hz sine wave).
|
||||
|
||||
Uses ``libopus`` as a fallback if ``libvorbis`` is unavailable in the
|
||||
bundled FFmpeg (e.g. PyAV wheels from an older build).
|
||||
"""
|
||||
buf = io.BytesIO()
|
||||
n = int(sample_rate * duration)
|
||||
samples = (np.sin(2 * np.pi * 440 * np.arange(n) / sample_rate) * 0.3).astype(np.float32)
|
||||
with av.open(buf, mode="w", format="ogg") as container:
|
||||
codec_name = "libopus" if "libvorbis" not in av.codecs_available else "libvorbis"
|
||||
stream = container.add_stream(codec_name, rate=sample_rate, layout="mono")
|
||||
frame = av.AudioFrame.from_ndarray(samples[np.newaxis, :], format="fltp", layout="mono")
|
||||
frame.sample_rate = sample_rate
|
||||
for pkt in stream.encode(frame):
|
||||
container.mux(pkt)
|
||||
for pkt in stream.encode():
|
||||
container.mux(pkt)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# video_duration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVideoDuration:
|
||||
@staticmethod
|
||||
def test_returns_duration(tmp_path: Path) -> None:
|
||||
mp4 = _make_synthetic_mp4(duration=2.0, fps=30)
|
||||
path = tmp_path / "test.mp4"
|
||||
path.write_bytes(mp4)
|
||||
dur = video_duration(path)
|
||||
assert abs(dur - 2.0) < 0.05, f"expected ~2.0s, got {dur}"
|
||||
|
||||
@staticmethod
|
||||
def test_raises_for_missing_file(tmp_path: Path) -> None:
|
||||
with pytest.raises((OSError, ValueError)):
|
||||
video_duration(tmp_path / "nonexistent.mp4")
|
||||
|
||||
@staticmethod
|
||||
def test_raises_for_broken_file(tmp_path: Path) -> None:
|
||||
broken = tmp_path / "broken.mp4"
|
||||
broken.write_bytes(b"\x00\x00\x00\x00")
|
||||
with pytest.raises((OSError, ValueError)):
|
||||
video_duration(broken)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_jpeg
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractJpeg:
|
||||
@staticmethod
|
||||
def test_returns_valid_jpeg(tmp_path: Path) -> None:
|
||||
mp4 = _make_synthetic_mp4(duration=1.0, fps=30)
|
||||
path = tmp_path / "test.mp4"
|
||||
path.write_bytes(mp4)
|
||||
jpeg = extract_jpeg(path, timestamp=0.5, quality=3)
|
||||
assert jpeg.startswith(b"\xff\xd8"), "not a valid JPEG header"
|
||||
assert len(jpeg) > 100
|
||||
|
||||
@staticmethod
|
||||
def test_quality_maps_to_pil_scale(tmp_path: Path) -> None:
|
||||
"""Higher ffmpeg quality (lower q value) produces a larger JPEG."""
|
||||
mp4 = _make_synthetic_mp4(duration=1.0, fps=30)
|
||||
path = tmp_path / "test.mp4"
|
||||
path.write_bytes(mp4)
|
||||
high = extract_jpeg(path, timestamp=0.5, quality=1)
|
||||
low = extract_jpeg(path, timestamp=0.5, quality=31)
|
||||
assert len(high) > len(low), "quality=1 should produce larger JPEG than quality=31"
|
||||
|
||||
@staticmethod
|
||||
def test_negative_timestamp_clamped(tmp_path: Path) -> None:
|
||||
"""Negative timestamps are clamped to 0."""
|
||||
mp4 = _make_synthetic_mp4(duration=1.0, fps=30)
|
||||
path = tmp_path / "test.mp4"
|
||||
path.write_bytes(mp4)
|
||||
jpeg = extract_jpeg(path, timestamp=-1.0, quality=3)
|
||||
assert jpeg.startswith(b"\xff\xd8")
|
||||
|
||||
@staticmethod
|
||||
def test_raises_value_error_for_no_video(tmp_path: Path) -> None:
|
||||
wav = _make_synthetic_wav()
|
||||
path = tmp_path / "test.wav"
|
||||
path.write_bytes(wav)
|
||||
with pytest.raises(ValueError):
|
||||
extract_jpeg(path, timestamp=0.0)
|
||||
|
||||
@staticmethod
|
||||
def test_long_gop_returns_different_frames_for_different_timestamps(tmp_path: Path) -> None:
|
||||
"""Long GOP: different timestamps must return different frame content (P2)."""
|
||||
# Create a long-GOP video: 3s @ 10fps = 30 frames, gop_size=30 (1 keyframe)
|
||||
buf = io.BytesIO()
|
||||
fps = 10
|
||||
duration = 3.0
|
||||
total_frames = int(duration * fps)
|
||||
with av.open(buf, mode="w", format="mp4") as container:
|
||||
stream = container.add_stream("mpeg4", rate=fps)
|
||||
stream.width = 64
|
||||
stream.height = 64
|
||||
stream.pix_fmt = "yuv420p"
|
||||
stream.gop_size = 30 # 3 seconds per keyframe
|
||||
for i in range(total_frames):
|
||||
frame = av.VideoFrame.from_ndarray(np.full((64, 64, 3), (i * 64) % 256, dtype=np.uint8), format="rgb24")
|
||||
for pkt in stream.encode(frame):
|
||||
container.mux(pkt)
|
||||
for pkt in stream.encode():
|
||||
container.mux(pkt)
|
||||
mp4 = buf.getvalue()
|
||||
|
||||
path = tmp_path / "long_gop.mp4"
|
||||
path.write_bytes(mp4)
|
||||
|
||||
# Extract frames at different timestamps
|
||||
frame0 = extract_jpeg(path, timestamp=0.0, quality=3)
|
||||
frame1 = extract_jpeg(path, timestamp=1.0, quality=3)
|
||||
frame2 = extract_jpeg(path, timestamp=2.0, quality=3)
|
||||
|
||||
# Verify all are valid JPEGs
|
||||
assert frame0.startswith(b"\xff\xd8")
|
||||
assert frame1.startswith(b"\xff\xd8")
|
||||
assert frame2.startswith(b"\xff\xd8")
|
||||
|
||||
# Verify content differs: decode JPEGs and compare mean brightness
|
||||
from PIL import Image
|
||||
|
||||
img0 = np.asarray(Image.open(io.BytesIO(frame0)))
|
||||
img1 = np.asarray(Image.open(io.BytesIO(frame1)))
|
||||
img2 = np.asarray(Image.open(io.BytesIO(frame2)))
|
||||
means = [img.mean() for img in (img0, img1, img2)]
|
||||
assert len(set(round(m, 1) for m in means)) >= 2, (
|
||||
f"timestamps 0/1/2s should give different frames, got means {means}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# read_audio_pcm16
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReadAudioPcm16:
|
||||
@staticmethod
|
||||
def test_wav_fast_path(tmp_path: Path) -> None:
|
||||
"""WAV with matching params uses the fast wave module path."""
|
||||
wav = _make_synthetic_wav(duration=1.0, sample_rate=16000)
|
||||
path = tmp_path / "test.wav"
|
||||
path.write_bytes(wav)
|
||||
pcm = read_audio_pcm16(path)
|
||||
assert len(pcm) == 16000 * 2, f"expected 32000 bytes, got {len(pcm)}"
|
||||
assert isinstance(pcm, bytes)
|
||||
|
||||
@staticmethod
|
||||
def test_wav_different_rate_resamples(tmp_path: Path) -> None:
|
||||
"""A 48 kHz WAV is resampled down to 16 kHz — exact sample count after flush."""
|
||||
wav = _make_synthetic_wav(duration=1.0, sample_rate=48000)
|
||||
path = tmp_path / "test.wav"
|
||||
path.write_bytes(wav)
|
||||
pcm = read_audio_pcm16(path)
|
||||
assert len(pcm) > 0
|
||||
assert len(pcm) % 2 == 0
|
||||
# 1s @ 48kHz → 1s @ 16kHz = 16000 samples = 32000 bytes (±1 sample)
|
||||
assert abs(len(pcm) - 32000) <= 4, f"expected ~32000 bytes, got {len(pcm)}"
|
||||
|
||||
@staticmethod
|
||||
def test_non_wav_ogg(tmp_path: Path) -> None:
|
||||
"""Non-WAV audio (OGG) is decoded and resampled via PyAV."""
|
||||
ogg = _make_synthetic_ogg(duration=1.0, sample_rate=16000)
|
||||
path = tmp_path / "test.ogg"
|
||||
path.write_bytes(ogg)
|
||||
pcm = read_audio_pcm16(path)
|
||||
assert len(pcm) > 0
|
||||
assert len(pcm) % 2 == 0
|
||||
|
||||
@staticmethod
|
||||
def test_non_wav_ogg_resample_accuracy(tmp_path: Path) -> None:
|
||||
"""OGG resample should produce exact sample count after flush."""
|
||||
ogg = _make_synthetic_ogg(duration=1.5, sample_rate=48000)
|
||||
path = tmp_path / "test.ogg"
|
||||
path.write_bytes(ogg)
|
||||
pcm = read_audio_pcm16(path)
|
||||
# 1.5s @ 48kHz → 1.5s @ 16kHz = 24000 samples = 48000 bytes (±1 sample)
|
||||
expected = int(1.5 * 16000) * 2
|
||||
assert abs(len(pcm) - expected) <= 4, f"expected ~{expected} bytes, got {len(pcm)}"
|
||||
|
||||
@staticmethod
|
||||
def test_raises_for_broken_file(tmp_path: Path) -> None:
|
||||
"""Corrupt audio must raise, not return empty bytes (L1)."""
|
||||
broken = tmp_path / "broken.mp3"
|
||||
broken.write_bytes(b"\x00\x00\x00\x00")
|
||||
with pytest.raises((OSError, ValueError)):
|
||||
read_audio_pcm16(broken)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _ffmpeg_q_to_pil_quality
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFfmpegQToPilQuality:
|
||||
@staticmethod
|
||||
def test_q1_maps_to_best() -> None:
|
||||
assert _ffmpeg_q_to_pil_quality(1) >= 90
|
||||
|
||||
@staticmethod
|
||||
def test_q31_maps_to_lowest() -> None:
|
||||
assert _ffmpeg_q_to_pil_quality(31) <= 10
|
||||
|
||||
@staticmethod
|
||||
def test_monotonic() -> None:
|
||||
"""Higher ffmpeg q → lower PIL quality (monotonically non-increasing)."""
|
||||
prev = 100
|
||||
for q in range(1, 32):
|
||||
p = _ffmpeg_q_to_pil_quality(q)
|
||||
assert p <= prev, f"quality.increase at q={q}: {prev} → {p}"
|
||||
prev = p
|
||||
|
||||
|
||||
def test_dict_with_bytes_materializes_a_real_file(tmp_path):
|
||||
# A Hugging Face Audio/Video feature value {"bytes": ..., "path": ...} used
|
||||
# to short-circuit on ``path`` (the artifact name inside the dataset, which
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Verify metric sample counts survive benchmark JSON serialization."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from vllm.benchmarks.datasets import SampleRequest
|
||||
from vllm.benchmarks.serve import TaskType
|
||||
|
||||
from vllm_omni.benchmarks.patch import patch
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selected_metrics", [["ttft", "tpot", "itl"], []])
|
||||
@pytest.mark.parametrize("measured", [True, False])
|
||||
def test_benchmark_result_preserves_tpot_sample_count(monkeypatch, measured, selected_metrics):
|
||||
async def request_func(**kwargs):
|
||||
return patch.MixRequestFuncOutput(
|
||||
success=True,
|
||||
prompt_len=2,
|
||||
output_tokens=3,
|
||||
generated_text="hello world!",
|
||||
start_time=time.perf_counter(),
|
||||
ttft=0.01,
|
||||
latency=0.03,
|
||||
text_latency=0.03,
|
||||
itl=[0.01, 0.01] if measured else [0.0, 0.0],
|
||||
)
|
||||
|
||||
monkeypatch.setitem(patch.ASYNC_REQUEST_FUNCS, "test-result", request_func)
|
||||
result = asyncio.run(
|
||||
patch.benchmark(
|
||||
task_type=TaskType.GENERATION,
|
||||
endpoint_type="test-result",
|
||||
api_url="http://unused/v1/chat/completions",
|
||||
base_url="http://unused",
|
||||
model_id="test-model",
|
||||
model_name="test-model",
|
||||
tokenizer=None,
|
||||
input_requests=[SampleRequest(prompt="hi", prompt_len=2, expected_output_len=3)],
|
||||
logprobs=None,
|
||||
request_rate=float("inf"),
|
||||
burstiness=1.0,
|
||||
disable_tqdm=True,
|
||||
num_warmups=0,
|
||||
profile=False,
|
||||
selected_percentile_metrics=selected_metrics,
|
||||
selected_percentiles=[50.0],
|
||||
ignore_eos=False,
|
||||
goodput_config_dict={},
|
||||
max_concurrency=1,
|
||||
lora_modules=None,
|
||||
extra_headers=None,
|
||||
extra_body=None,
|
||||
ready_check_timeout_sec=0,
|
||||
)
|
||||
)
|
||||
# DFX reads this dictionary from the benchmark's JSON result file.
|
||||
saved_result = json.loads(json.dumps(result))
|
||||
assert saved_result["completed"] == 1
|
||||
assert saved_result.get("num_tpot_samples") == int(measured)
|
||||
assert saved_result["num_ttft_samples"] == 1
|
||||
assert saved_result["num_itl_samples"] == 2
|
||||
assert "num_audio_ttfp_samples" in saved_result
|
||||
assert "num_audio_rtf_samples" in saved_result
|
||||
if measured and "tpot" in selected_metrics:
|
||||
assert saved_result["mean_tpot_ms"] == pytest.approx(10.0)
|
||||
else:
|
||||
assert "mean_tpot_ms" not in saved_result
|
||||
@@ -47,11 +47,50 @@ from benchmarks.accuracy.text_to_image.gbench import (
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_serve_common_argv,
|
||||
seed_tts_bench_argv,
|
||||
videomme_bench_argv,
|
||||
)
|
||||
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
|
||||
|
||||
|
||||
def test_videomme_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", "lmms-eval/Video-MME")
|
||||
monkeypatch.delenv("VLLM_VIDEOMME_REPO", raising=False)
|
||||
monkeypatch.delenv("VIDEOMME_ROOT", raising=False)
|
||||
|
||||
argv = videomme_bench_argv()
|
||||
|
||||
dataset_idx = argv.index("--dataset-path")
|
||||
assert argv[argv.index("--dataset-name") + 1] == "videomme"
|
||||
assert argv[dataset_idx + 1] == "lmms-eval/Video-MME"
|
||||
|
||||
|
||||
def test_videomme_bench_argv_uses_local_directory(monkeypatch, tmp_path: Path):
|
||||
monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", str(tmp_path))
|
||||
monkeypatch.delenv("VIDEOMME_ROOT", raising=False)
|
||||
|
||||
argv = videomme_bench_argv()
|
||||
|
||||
assert argv[argv.index("--dataset-path") + 1] == str(tmp_path.resolve())
|
||||
|
||||
|
||||
def test_sync_dataset_env_preserves_videomme_hf_repo_id(monkeypatch):
|
||||
ns = argparse.Namespace(
|
||||
daily_omni_repo=None,
|
||||
daily_omni_qa_json=None,
|
||||
daily_omni_video_dir=None,
|
||||
videomme_repo=None,
|
||||
videomme_dataset_path="lmms-eval/Video-MME",
|
||||
seed_tts_dataset_path=None,
|
||||
seed_tts_root=None,
|
||||
)
|
||||
|
||||
monkeypatch.delenv("VLLM_VIDEOMME_DATASET_PATH", raising=False)
|
||||
sync_dataset_env_from_ns(ns)
|
||||
|
||||
assert os.environ["VLLM_VIDEOMME_DATASET_PATH"] == "lmms-eval/Video-MME"
|
||||
|
||||
|
||||
def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_SEED_TTS_DATASET_PATH", "zhaochenyang20/seed-tts-eval")
|
||||
monkeypatch.delenv("VLLM_SEED_TTS_REPO", raising=False)
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from benchmarks.diffusion import backends
|
||||
from benchmarks.diffusion.backends import RequestFuncInput, RequestFuncOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu, pytest.mark.asyncio]
|
||||
|
||||
ENDPOINT = "/v1/chat/completions"
|
||||
TEST_MODEL = "test-model"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def diffusion_benchmark(monkeypatch):
|
||||
# The standalone script imports its sibling backend module by name.
|
||||
monkeypatch.setitem(sys.modules, "backends", backends)
|
||||
return importlib.import_module("benchmarks.diffusion.diffusion_benchmark_serving")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def warmup_args(diffusion_benchmark):
|
||||
from benchmarks.diffusion.diffusion_benchmark_serving import build_parser
|
||||
|
||||
parser = build_parser()
|
||||
return parser.parse_args(
|
||||
[
|
||||
f"--endpoint={ENDPOINT}",
|
||||
"--dataset=random",
|
||||
"--task=t2i",
|
||||
"--warmup-requests=2",
|
||||
"--warmup-num-inference-steps=4",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def input_requests():
|
||||
return [
|
||||
RequestFuncInput(
|
||||
prompt=prompt,
|
||||
api_url=f"http://test.local{ENDPOINT}",
|
||||
model=TEST_MODEL,
|
||||
num_inference_steps=20,
|
||||
)
|
||||
for prompt in ("first", "second")
|
||||
]
|
||||
|
||||
|
||||
async def test_disabled_warmup_sends_no_requests(diffusion_benchmark, warmup_args, input_requests):
|
||||
warmup_args.warmup_requests = 0
|
||||
|
||||
async def unexpected_request(*args):
|
||||
pytest.fail("disabled warmup must not send requests")
|
||||
|
||||
pairs = await diffusion_benchmark._run_warmups(input_requests, warmup_args, None, unexpected_request)
|
||||
|
||||
assert pairs == []
|
||||
|
||||
|
||||
async def test_successful_warmups_preserve_request_output_pairs(diffusion_benchmark, warmup_args, input_requests):
|
||||
outputs = [RequestFuncOutput(success=True, latency=0.5), RequestFuncOutput(success=True, latency=1.0)]
|
||||
pending_outputs = iter(outputs)
|
||||
|
||||
async def successful_request(req, session, pbar):
|
||||
return next(pending_outputs)
|
||||
|
||||
pairs = await diffusion_benchmark._run_warmups(input_requests, warmup_args, None, successful_request)
|
||||
|
||||
assert [request.prompt for request, _ in pairs] == ["first", "second"]
|
||||
assert [request.num_inference_steps for request, _ in pairs] == [4, 4]
|
||||
assert [output for _, output in pairs] == outputs
|
||||
assert [request.num_inference_steps for request in input_requests] == [20, 20]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed_count", [1, 2], ids=["partial-failure", "all-failed"])
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
["HTTP 503: private prompt rejected", "Cannot connect to https://test.local/?token=private"],
|
||||
ids=["http-error", "transport-error"],
|
||||
)
|
||||
async def test_failed_warmups_abort_without_exposing_response_errors(
|
||||
diffusion_benchmark, warmup_args, input_requests, failed_count, error
|
||||
):
|
||||
outputs = iter(
|
||||
[RequestFuncOutput(success=False, error=error) for _ in range(failed_count)]
|
||||
+ [RequestFuncOutput(success=True) for _ in range(2 - failed_count)]
|
||||
)
|
||||
|
||||
async def request(req, session, pbar):
|
||||
return next(outputs)
|
||||
|
||||
with pytest.raises(RuntimeError, match=f"{failed_count}/2 warmup requests failed") as exc_info:
|
||||
await diffusion_benchmark._run_warmups(input_requests, warmup_args, None, request)
|
||||
|
||||
assert error not in str(exc_info.value)
|
||||
assert "server logs" in str(exc_info.value)
|
||||
|
||||
|
||||
async def test_benchmark_does_not_start_measurement_after_failed_warmup(
|
||||
monkeypatch, mocker, diffusion_benchmark, warmup_args, input_requests
|
||||
):
|
||||
sent = []
|
||||
|
||||
async def failed_request(req, session, pbar):
|
||||
sent.append(req)
|
||||
return RequestFuncOutput(success=False, error="HTTP 503: unavailable")
|
||||
|
||||
monkeypatch.setitem(
|
||||
diffusion_benchmark.backends_function_mapping["2i"],
|
||||
warmup_args.endpoint,
|
||||
(failed_request, warmup_args.endpoint),
|
||||
)
|
||||
dataset = mocker.Mock(spec=diffusion_benchmark.RandomDataset)
|
||||
dataset.get_requests.return_value = input_requests
|
||||
monkeypatch.setattr(diffusion_benchmark, "RandomDataset", mocker.Mock(return_value=dataset))
|
||||
timer = mocker.Mock(spec=diffusion_benchmark.time)
|
||||
timer.perf_counter.side_effect = AssertionError("measurement started after failed warmup")
|
||||
monkeypatch.setattr(diffusion_benchmark, "time", timer)
|
||||
|
||||
with pytest.raises(RuntimeError, match="2/2 warmup requests failed"):
|
||||
await diffusion_benchmark.benchmark(warmup_args)
|
||||
|
||||
assert len(sent) == warmup_args.warmup_requests
|
||||
@@ -0,0 +1,546 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Offline Video-MME sources: local mirrors must not fall back to a Hub id."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import zipfile
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import _validate_videomme
|
||||
from vllm_omni.benchmarks.data_modules.videomme_dataset import (
|
||||
VideoMMEDataset,
|
||||
VideoMMESampleRequest,
|
||||
cached_videomme_snapshots,
|
||||
ensure_videomme_hub_root,
|
||||
ensure_videomme_videos_extracted,
|
||||
resolve_videomme_local_root,
|
||||
videomme_local_parquet,
|
||||
videomme_local_subtitle_dir,
|
||||
videomme_local_video_dir,
|
||||
)
|
||||
from vllm_omni.benchmarks.data_modules.videomme_eval import (
|
||||
compute_videomme_accuracy_metrics,
|
||||
extract_characters_regex,
|
||||
normalize_gold_answer,
|
||||
)
|
||||
from vllm_omni.benchmarks.patch.patch import _looks_like_hf_dataset_id, _videomme_repo_from_args, get_samples
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
|
||||
|
||||
|
||||
def _write_parquet_placeholder(root: Path) -> Path:
|
||||
path = root / "videomme" / "test-00000-of-00001.parquet"
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_bytes(b"parquet")
|
||||
return path
|
||||
|
||||
|
||||
def test_resolve_local_root_returns_none_for_hub_id() -> None:
|
||||
assert resolve_videomme_local_root("lmms-eval/Video-MME") is None
|
||||
assert resolve_videomme_local_root("") is None
|
||||
assert resolve_videomme_local_root(None) is None
|
||||
|
||||
|
||||
def test_plain_local_mirror(tmp_path: Path) -> None:
|
||||
pq = _write_parquet_placeholder(tmp_path)
|
||||
video = tmp_path / "video" / "abc.mp4"
|
||||
video.parent.mkdir()
|
||||
video.write_bytes(b"fake")
|
||||
(tmp_path / "subtitle").mkdir()
|
||||
(tmp_path / "subtitle" / "abc.srt").write_text("1\n", encoding="utf-8")
|
||||
|
||||
root = resolve_videomme_local_root(str(tmp_path))
|
||||
assert root == tmp_path.resolve()
|
||||
assert videomme_local_parquet(root) == pq.resolve()
|
||||
assert videomme_local_video_dir(root) == tmp_path.resolve() / "video"
|
||||
assert videomme_local_subtitle_dir(root) == tmp_path.resolve() / "subtitle"
|
||||
|
||||
|
||||
def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
|
||||
cache_dir = tmp_path / "datasets--lmms-eval--Video-MME"
|
||||
snapshot = cache_dir / "snapshots" / "deadbeef"
|
||||
snapshot.mkdir(parents=True)
|
||||
_write_parquet_placeholder(snapshot)
|
||||
(cache_dir / "refs").mkdir()
|
||||
(cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
|
||||
|
||||
root = resolve_videomme_local_root(str(cache_dir))
|
||||
assert root == snapshot.resolve()
|
||||
assert videomme_local_parquet(root) is not None
|
||||
|
||||
|
||||
def test_hf_cache_dir_with_unusable_ref_falls_back_to_revision(tmp_path: Path) -> None:
|
||||
cache_dir = tmp_path / "datasets--lmms-eval--Video-MME"
|
||||
snapshot = cache_dir / "snapshots" / "deadbeef"
|
||||
snapshot.mkdir(parents=True)
|
||||
_write_parquet_placeholder(snapshot)
|
||||
(cache_dir / "refs").mkdir()
|
||||
# An empty ref must not resolve to snapshots/, which holds no dataset files.
|
||||
(cache_dir / "refs" / "main").write_text("", encoding="utf-8")
|
||||
|
||||
assert resolve_videomme_local_root(str(cache_dir)) == snapshot.resolve()
|
||||
|
||||
|
||||
def _stub_failing_hub(monkeypatch: pytest.MonkeyPatch, hub_cache: Path) -> dict[str, object]:
|
||||
"""Point the Hub cache at ``hub_cache`` and make every snapshot_download miss."""
|
||||
from huggingface_hub import constants as hf_constants
|
||||
from huggingface_hub.errors import LocalEntryNotFoundError
|
||||
|
||||
from vllm_omni.transformers_utils import repo_utils
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
class _Api:
|
||||
def snapshot_download(self, **kwargs: object) -> str:
|
||||
seen.update(kwargs)
|
||||
raise LocalEntryNotFoundError("no cached snapshot")
|
||||
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_OFFLINE", True)
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(hub_cache))
|
||||
monkeypatch.setattr(repo_utils, "hf_api", lambda: _Api())
|
||||
return seen
|
||||
|
||||
|
||||
def test_hub_root_offline_without_cache_points_at_local_options(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
seen = _stub_failing_hub(monkeypatch, tmp_path)
|
||||
|
||||
with pytest.raises(FileNotFoundError, match="--videomme-video-dir") as excinfo:
|
||||
ensure_videomme_hub_root("lmms-lab/Video-MME")
|
||||
|
||||
# Offline runs must consult the cache directly instead of resolving over the network.
|
||||
assert seen["local_files_only"] is True
|
||||
assert "already cached under another repo id" not in str(excinfo.value)
|
||||
|
||||
|
||||
def test_hub_root_offline_names_snapshot_cached_under_another_repo_id(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
sibling = tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef"
|
||||
sibling.mkdir(parents=True)
|
||||
_write_parquet_placeholder(sibling)
|
||||
_stub_failing_hub(monkeypatch, tmp_path)
|
||||
|
||||
with pytest.raises(FileNotFoundError) as excinfo:
|
||||
ensure_videomme_hub_root("lmms-lab/Video-MME")
|
||||
|
||||
assert str(sibling.resolve()) in str(excinfo.value)
|
||||
|
||||
|
||||
def test_cached_videomme_snapshots_skips_partial_download(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
(tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef").mkdir(parents=True)
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
|
||||
|
||||
assert cached_videomme_snapshots() == []
|
||||
|
||||
|
||||
def test_extract_videos_rejects_empty_tree(tmp_path: Path) -> None:
|
||||
(tmp_path / "video").mkdir()
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
ensure_videomme_videos_extracted(tmp_path)
|
||||
assert not (tmp_path / ".videomme_videos_extracted").exists()
|
||||
|
||||
|
||||
def test_nested_unzipped_videos_are_discovered(tmp_path: Path) -> None:
|
||||
nested = tmp_path / "videos" / "videos_chunked_01" / "data" / "xyz.mp4"
|
||||
nested.parent.mkdir(parents=True)
|
||||
nested.write_bytes(b"fake")
|
||||
|
||||
assert videomme_local_video_dir(tmp_path) == tmp_path / "videos"
|
||||
|
||||
|
||||
def test_extract_characters_regex_matches_official_prefixes() -> None:
|
||||
assert extract_characters_regex("The best answer is B.") == "B"
|
||||
assert extract_characters_regex("Answer: C") == "C"
|
||||
assert extract_characters_regex("I think D is correct") == "D"
|
||||
assert extract_characters_regex("no letter here at all in this long sentence") is None
|
||||
|
||||
|
||||
def test_normalize_gold_answer() -> None:
|
||||
assert normalize_gold_answer("a") == "A"
|
||||
assert normalize_gold_answer("B. content") == "B"
|
||||
assert normalize_gold_answer("") is None
|
||||
|
||||
|
||||
class _Out:
|
||||
def __init__(self, text: str, *, success: bool = True) -> None:
|
||||
self.generated_text = text
|
||||
self.success = success
|
||||
self.error = "" if success else "http fail"
|
||||
|
||||
|
||||
def test_compute_videomme_accuracy_metrics() -> None:
|
||||
reqs = [
|
||||
VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="A",
|
||||
videomme_video_id="v1",
|
||||
videomme_question_id="q1",
|
||||
videomme_duration="short",
|
||||
videomme_domain="Knowledge",
|
||||
videomme_sub_category="Science",
|
||||
videomme_task_type="QA",
|
||||
),
|
||||
VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="B",
|
||||
videomme_video_id="v2",
|
||||
videomme_question_id="q2",
|
||||
videomme_duration="long",
|
||||
videomme_domain="Knowledge",
|
||||
videomme_sub_category="Science",
|
||||
videomme_task_type="QA",
|
||||
),
|
||||
]
|
||||
metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("The answer is C")])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_correct"] == 1
|
||||
assert metrics["videomme_evaluated_ok"] == 2
|
||||
assert metrics["videomme_accuracy"] == 0.5
|
||||
assert metrics["videomme_submitted"] == 2
|
||||
assert metrics["videomme_unique_question_ids"] == 2
|
||||
assert _validate_videomme(metrics, min_accuracy=0.4) == []
|
||||
assert "videomme_eval_items" not in metrics
|
||||
assert metrics["videomme_per_duration_accuracy"]["short"] == 1.0
|
||||
assert metrics["videomme_per_duration_accuracy"]["long"] == 0.0
|
||||
|
||||
|
||||
def test_compute_videomme_accuracy_metrics_http_fail_excluded_from_default() -> None:
|
||||
reqs = [
|
||||
VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="A",
|
||||
videomme_question_id="q1",
|
||||
),
|
||||
VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="B",
|
||||
videomme_question_id="q2",
|
||||
),
|
||||
]
|
||||
metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("", success=False)])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_accuracy"] == 1.0
|
||||
assert metrics["videomme_accuracy_incl_http_fail"] == 0.5
|
||||
assert metrics["videomme_request_failed"] == 1
|
||||
errs = _validate_videomme(metrics, min_accuracy=0.68)
|
||||
assert any("videomme_request_failed" in err for err in errs)
|
||||
|
||||
|
||||
def test_compute_videomme_accuracy_metrics_saves_eval_items() -> None:
|
||||
req = VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="A",
|
||||
videomme_question_id="q1",
|
||||
)
|
||||
metrics = compute_videomme_accuracy_metrics([req], [_Out("A")], include_per_item=True)
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_eval_items"][0]["question_id"] == "q1"
|
||||
|
||||
|
||||
def test_videomme_save_eval_items_cli_sets_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from vllm_omni.benchmarks import serve
|
||||
|
||||
seen: dict[str, str | None] = {}
|
||||
|
||||
async def fake_main_async(args: Namespace) -> dict[str, str]:
|
||||
seen["env"] = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS")
|
||||
return {"ok": "1"}
|
||||
|
||||
monkeypatch.delenv("VIDEOMME_SAVE_EVAL_ITEMS", raising=False)
|
||||
monkeypatch.setattr(serve, "main_async", fake_main_async)
|
||||
|
||||
args = Namespace(
|
||||
videomme_save_eval_items=True,
|
||||
seed_tts_wer_eval=False,
|
||||
seed_tts_wer_save_items=False,
|
||||
daily_omni_save_eval_items=False,
|
||||
omni_request_timeout_s=None,
|
||||
endpoint=None,
|
||||
backend="openai-chat-omni",
|
||||
explicit_keys=frozenset(),
|
||||
extra_body=None,
|
||||
print_stage=False,
|
||||
dataset_name="videomme",
|
||||
)
|
||||
assert serve.main(args) == {"ok": "1"}
|
||||
assert seen["env"] == "1"
|
||||
|
||||
|
||||
def _file_uri_path(url: str) -> Path:
|
||||
parsed = urlparse(url)
|
||||
assert parsed.scheme == "file"
|
||||
return Path(unquote(parsed.path))
|
||||
|
||||
|
||||
def test_relative_video_dir_emits_absolute_file_uris_cold_and_warm(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
videos = tmp_path / "videos"
|
||||
videos.mkdir()
|
||||
(videos / "vid1.mp4").write_bytes(b"fake")
|
||||
|
||||
def fake_extract(cls, video_path, *, include_audio, max_num_frames): # noqa: ANN001
|
||||
return [Image.new("RGB", (4, 4), color="red")], []
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None)
|
||||
monkeypatch.setattr(VideoMMEDataset, "_extract_frames_and_audio", classmethod(fake_extract))
|
||||
|
||||
ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1)
|
||||
assert ds.video_dir == videos.resolve()
|
||||
|
||||
cold = ds._get_minicpm_frame_parts("vid1", include_audio=False)
|
||||
assert cold is not None
|
||||
cold_path = _file_uri_path(cold[0]["image_url"]["url"])
|
||||
assert cold_path.is_absolute()
|
||||
assert cold_path.is_file()
|
||||
|
||||
warm_ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1)
|
||||
warm = warm_ds._get_minicpm_frame_parts("vid1", include_audio=False)
|
||||
assert warm is not None
|
||||
warm_path = _file_uri_path(warm[0]["image_url"]["url"])
|
||||
assert warm_path.is_absolute()
|
||||
assert warm_path == cold_path
|
||||
|
||||
|
||||
def test_videomme_repo_from_args_honors_custom_hub_id() -> None:
|
||||
args = Namespace(dataset_name="videomme", dataset_path="my-org/custom-videomme", hf_name=None)
|
||||
assert _videomme_repo_from_args(args, explicit=True) == "my-org/custom-videomme"
|
||||
|
||||
|
||||
def test_videomme_repo_from_args_defaults_when_path_omitted() -> None:
|
||||
args = Namespace(dataset_name="videomme", dataset_path=None, hf_name=None)
|
||||
assert _videomme_repo_from_args(args, explicit=True) is None
|
||||
|
||||
|
||||
def test_videomme_repo_from_args_rejects_unsupported_path() -> None:
|
||||
args = Namespace(dataset_name="videomme", dataset_path="not-a-local-dir-or-hub-id", hf_name=None)
|
||||
with pytest.raises(ValueError, match="Unsupported Video-MME"):
|
||||
_videomme_repo_from_args(args, explicit=True)
|
||||
|
||||
|
||||
def test_get_samples_rejects_unsupported_videomme_dataset_path() -> None:
|
||||
args = Namespace(
|
||||
dataset_name="videomme",
|
||||
dataset_path="not-a-local-dir-or-hub-id",
|
||||
hf_name=None,
|
||||
backend="openai-chat-omni",
|
||||
seed=0,
|
||||
)
|
||||
with pytest.raises(ValueError, match="Unsupported Video-MME"):
|
||||
get_samples(args, None)
|
||||
|
||||
|
||||
def test_hf_dataset_name_does_not_auto_detect_custom_repo() -> None:
|
||||
args = Namespace(dataset_name="hf", dataset_path="my-org/custom-videomme", hf_name=None)
|
||||
assert _videomme_repo_from_args(args) is None
|
||||
assert _looks_like_hf_dataset_id("my-org/custom-videomme")
|
||||
assert not _looks_like_hf_dataset_id("./videos")
|
||||
official = Namespace(dataset_name="hf", dataset_path="lmms-eval/Video-MME", hf_name=None)
|
||||
assert _videomme_repo_from_args(official) == "lmms-eval/Video-MME"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_frames", [0, -1])
|
||||
def test_rejects_nonpositive_max_frames(max_frames: int, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None)
|
||||
with pytest.raises(ValueError, match="max_frames must be positive"):
|
||||
VideoMMEDataset(parquet_path="unused.parquet", max_frames=max_frames)
|
||||
|
||||
|
||||
def test_frame_cache_rebuild_overwrites_incomplete_files(tmp_path: Path) -> None:
|
||||
frame = tmp_path / "frame_0000.jpg"
|
||||
frame.write_bytes(b"interrupted JPEG")
|
||||
VideoMMEDataset._emit_media(tmp_path, frame.name, b"complete JPEG", "image_url", "image/jpeg")
|
||||
assert frame.read_bytes() == b"complete JPEG"
|
||||
|
||||
|
||||
def test_frame_cache_rejects_empty_media(tmp_path: Path) -> None:
|
||||
(tmp_path / "manifest.json").write_text('{"count": 1}', encoding="utf-8")
|
||||
(tmp_path / "frame_0000.jpg").touch()
|
||||
dataset = object.__new__(VideoMMEDataset)
|
||||
assert dataset._cached_parts_from_disk(tmp_path, include_audio=False) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["duplicates", "missing_gold"])
|
||||
def test_accuracy_gate_rejects_incomplete_question_coverage(case: str) -> None:
|
||||
def request(question_id: str, gold: str) -> VideoMMESampleRequest:
|
||||
return VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_question_id=question_id,
|
||||
videomme_gold_answer=gold,
|
||||
)
|
||||
|
||||
requests = [
|
||||
request("q1", "A"),
|
||||
request("q1" if case == "duplicates" else "q2", "" if case == "missing_gold" else "A"),
|
||||
]
|
||||
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A"), _Out("A")])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_accuracy"] == 1.0
|
||||
errors = _validate_videomme(metrics, min_accuracy=0.68)
|
||||
assert any(("unique" if case == "duplicates" else "no_gold") in error for error in errors)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def videomme_mirror(tmp_path: Path) -> Path:
|
||||
import pandas as pd
|
||||
|
||||
parquet = tmp_path / "videomme" / "test-00000-of-00001.parquet"
|
||||
parquet.parent.mkdir()
|
||||
pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"videoID": f"v{i}",
|
||||
"question_id": f"q{i}",
|
||||
"question": "Which option is correct?",
|
||||
"options": ["A. first", "B. second", "C. third", "D. fourth"],
|
||||
"answer": "A",
|
||||
"duration": duration,
|
||||
}
|
||||
for i, duration in enumerate(("short", "long"), start=1)
|
||||
]
|
||||
).to_parquet(parquet)
|
||||
videos = tmp_path / "video"
|
||||
videos.mkdir()
|
||||
for i in (1, 2):
|
||||
(videos / f"v{i}.mp4").write_bytes(b"video")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _sample_video_urls(dataset: VideoMMEDataset, monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> list:
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer
|
||||
)
|
||||
return dataset.sample(Namespace(encode=lambda text: [1]), **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("no_oversample", [True, False])
|
||||
def test_missing_video_cannot_pass_accuracy_gate(
|
||||
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch, no_oversample: bool
|
||||
) -> None:
|
||||
(videomme_mirror / "video" / "v2.mp4").unlink()
|
||||
dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", disable_shuffle=True)
|
||||
requests = _sample_video_urls(dataset, monkeypatch, num_requests=2, no_oversample=no_oversample)
|
||||
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_accuracy"] == 1.0
|
||||
assert metrics["videomme_submitted"] == (1 if no_oversample else 2)
|
||||
assert metrics["videomme_skipped_rows"] == 1
|
||||
assert any("videomme_skipped_rows" in error for error in _validate_videomme(metrics, min_accuracy=0.68))
|
||||
|
||||
|
||||
def test_complete_duration_subset_can_pass_accuracy_gate(
|
||||
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", duration_filter="short")
|
||||
requests = _sample_video_urls(dataset, monkeypatch, num_requests=2700, no_oversample=True)
|
||||
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_submitted"] == 1
|
||||
assert metrics["videomme_skipped_rows"] == 0
|
||||
assert _validate_videomme(metrics, min_accuracy=0.68) == []
|
||||
|
||||
|
||||
def test_dataset_direct_hub_loading_resolves_video_assets(
|
||||
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
downloaded = []
|
||||
|
||||
def download(repo_id: str) -> Path:
|
||||
downloaded.append(repo_id)
|
||||
return videomme_mirror
|
||||
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.ensure_videomme_hub_root", download)
|
||||
dataset = VideoMMEDataset(dataset_path="my-org/custom-videomme", pack_mode="video_url")
|
||||
assert downloaded == ["my-org/custom-videomme"]
|
||||
assert dataset.video_dir == videomme_mirror / "video"
|
||||
assert len(dataset.data) == 2
|
||||
|
||||
|
||||
def test_local_source_with_explicit_parquet_keeps_video_root(videomme_mirror: Path) -> None:
|
||||
dataset = VideoMMEDataset(
|
||||
dataset_path=str(videomme_mirror),
|
||||
parquet_path=str(videomme_local_parquet(videomme_mirror)),
|
||||
pack_mode="video_url",
|
||||
)
|
||||
assert dataset.video_dir == videomme_mirror / "video"
|
||||
assert len(dataset.data) == 2
|
||||
|
||||
|
||||
def test_local_source_without_parquet_fails_locally(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def unexpected_hub(*args, **kwargs):
|
||||
pytest.fail("An explicitly selected local mirror must not fall back to Hub loading")
|
||||
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.load_dataset", unexpected_hub)
|
||||
with pytest.raises(FileNotFoundError, match="No Video-MME parquet"):
|
||||
VideoMMEDataset(dataset_path=str(tmp_path))
|
||||
|
||||
|
||||
def test_get_samples_prefers_local_root_over_hf_name(videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
args = Namespace(
|
||||
dataset_name="videomme",
|
||||
dataset_path=str(videomme_mirror),
|
||||
hf_name="unused-hf-name",
|
||||
backend="openai-chat-omni",
|
||||
seed=0,
|
||||
videomme_pack_mode="video_url",
|
||||
num_prompts=2,
|
||||
request_id_prefix="",
|
||||
no_oversample=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer
|
||||
)
|
||||
requests = get_samples(args, Namespace(encode=lambda text: [1]))
|
||||
assert len(requests) == 2
|
||||
assert {request.videomme_question_id for request in requests} == {"q1", "q2"}
|
||||
|
||||
|
||||
def test_extraction_replaces_truncated_video(tmp_path: Path) -> None:
|
||||
video = tmp_path / "video" / "v1.mp4"
|
||||
video.parent.mkdir()
|
||||
video.write_bytes(b"partial")
|
||||
with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive:
|
||||
archive.writestr("nested/v1.mp4", b"complete video")
|
||||
assert ensure_videomme_videos_extracted(tmp_path) == video.parent
|
||||
assert video.read_bytes() == b"complete video"
|
||||
|
||||
|
||||
def test_failed_extraction_does_not_publish_partial_video(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive:
|
||||
archive.writestr("v1.mp4", b"complete video")
|
||||
|
||||
def interrupted_copy(src, dst):
|
||||
dst.write(b"partial")
|
||||
raise OSError("interrupted")
|
||||
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.shutil.copyfileobj", interrupted_copy)
|
||||
with pytest.raises(OSError, match="interrupted"):
|
||||
ensure_videomme_videos_extracted(tmp_path)
|
||||
assert list((tmp_path / "video").iterdir()) == []
|
||||
assert not (tmp_path / ".videomme_videos_extracted").exists()
|
||||
@@ -11,6 +11,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
AMD_MERGE_PIPELINE = Path(".buildkite/amd/test-amd-merge.yml")
|
||||
AMD_NIGHTLY_PIPELINE = Path(".buildkite/amd/test-amd-nightly.yml")
|
||||
AMD_READY_PIPELINE = Path(".buildkite/amd/test-amd-ready.yml")
|
||||
AMD_TEMPLATE = Path(".buildkite/amd/test-template-amd-omni.j2")
|
||||
|
||||
|
||||
def _find_step(label: str, pipeline_path: Path = AMD_MERGE_PIPELINE) -> dict:
|
||||
@@ -53,3 +55,25 @@ def test_qwen3_accuracy_defers_artifact_path_expansion() -> None:
|
||||
assert '"$${BUILDKITE_BUILD_CHECKOUT_PATH:?}"' in staging_command
|
||||
assert '"$$artifact_dir"' in staging_command
|
||||
assert step["artifact_paths"] == ["tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"]
|
||||
|
||||
|
||||
def test_ready_diffusion_cpu_suite_is_sharded() -> None:
|
||||
step = _find_step("Simple · Diffusion Test · Shard %N/%t", AMD_READY_PIPELINE)
|
||||
pytest_command = next(command for command in step["commands"] if "pytest" in command)
|
||||
|
||||
assert step["parallelism"] == 4
|
||||
assert step["timeout_in_minutes"] == 45
|
||||
assert "--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT" in pytest_command
|
||||
assert "--shard-id=$$BUILDKITE_PARALLEL_JOB" in pytest_command
|
||||
|
||||
|
||||
def test_cosyvoice_gpu_abort_gets_one_fresh_job_retry() -> None:
|
||||
step = _find_step("CosyVoice3-TTS E2E Test", AMD_READY_PIPELINE)
|
||||
|
||||
assert step["retry"] == {"automatic": [{"exit_status": 134, "limit": 1}]}
|
||||
|
||||
template = AMD_TEMPLATE.read_text(encoding="utf-8")
|
||||
# Both grouped and top-level AMD steps must preserve an explicit retry
|
||||
# policy when the source suite is rendered into the uploaded pipeline.
|
||||
assert template.count("{% if step.retry %}") == 2
|
||||
assert template.count("{% for retry_rule in step.retry.automatic %}") == 2
|
||||
@@ -9,8 +9,10 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
AMD_TEMPLATE = REPO_ROOT / ".buildkite/amd/test-template-amd-omni.j2"
|
||||
AMD_BUILD_SCRIPT = REPO_ROOT / ".buildkite/amd/scripts/build-ci-image.sh"
|
||||
CI_DOCKERFILE = REPO_ROOT / "docker/Dockerfile.ci"
|
||||
ROCM_DOCKERFILE = REPO_ROOT / "docker/Dockerfile.rocm"
|
||||
ROCM_DOCKERIGNORE = REPO_ROOT / "docker/Dockerfile.rocm.dockerignore"
|
||||
|
||||
|
||||
def _docker_arg(path: Path, name: str) -> str:
|
||||
@@ -42,16 +44,63 @@ def test_rocm_defaults_to_prebuilt_base_image() -> None:
|
||||
assert _docker_arg(ROCM_DOCKERFILE, "USE_NIGHTLY_BUILD") == "0"
|
||||
|
||||
|
||||
def test_amd_build_uses_rocm_dockerfile_defaults() -> None:
|
||||
def test_amd_build_uses_registry_cache() -> None:
|
||||
template = AMD_TEMPLATE.read_text(encoding="utf-8")
|
||||
build_commands = [line.strip() for line in template.splitlines() if '"docker build ' in line]
|
||||
script_command = '"bash .buildkite/amd/scripts/build-ci-image.sh"'
|
||||
script = AMD_BUILD_SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert len(build_commands) == 1, f"expected one AMD image build command, found {len(build_commands)}"
|
||||
build_command = build_commands[0]
|
||||
assert "-f docker/Dockerfile.rocm" in build_command
|
||||
assert template.count(script_command) == 1
|
||||
assert "docker buildx build --push" in script
|
||||
assert '--cache-from "type=registry,ref=${CACHE_REF}"' in script
|
||||
assert '--cache-to "type=registry,ref=${CACHE_REF}' in script
|
||||
assert "mode=max" in script
|
||||
|
||||
|
||||
def test_amd_cache_key_tracks_dependency_inputs_and_architecture() -> None:
|
||||
script = AMD_BUILD_SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
for cache_input in (
|
||||
"docker/Dockerfile.rocm",
|
||||
"${DOCKERFILE}.dockerignore",
|
||||
"pyproject.toml",
|
||||
"setup.py",
|
||||
"requirements",
|
||||
"tools/install_torchcodec_rocm.sh",
|
||||
"ROCM_ARCH",
|
||||
):
|
||||
assert cache_input in script
|
||||
|
||||
|
||||
def test_amd_build_uses_rocm_dockerfile_defaults() -> None:
|
||||
build_script = AMD_BUILD_SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'DOCKERFILE="docker/Dockerfile.rocm"' in build_script
|
||||
for arg_name in ("BASE_IMAGE", "USE_NIGHTLY_BUILD"):
|
||||
assert f"--build-arg {arg_name}" not in build_command
|
||||
assert f"--build-arg={arg_name}" not in build_command
|
||||
assert f"--build-arg {arg_name}" not in build_script
|
||||
assert f"--build-arg={arg_name}" not in build_script
|
||||
|
||||
|
||||
def test_rocm_source_copy_preserves_dependency_cache() -> None:
|
||||
dockerfile = ROCM_DOCKERFILE.read_text(encoding="utf-8")
|
||||
lines = dockerfile.splitlines()
|
||||
source_copy_index = _line_index(lines, "COPY --link . .")
|
||||
test_stage_index = _line_index(lines, "FROM base AS test")
|
||||
|
||||
assert "FROM deps AS base" in lines[:source_copy_index]
|
||||
assert any('-e ".[dev]"' in line for line in lines[:source_copy_index])
|
||||
assert not any(
|
||||
line.startswith(("RUN ", "COPY ", "ADD ")) for line in lines[source_copy_index + 1 : test_stage_index]
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_build_context_excludes_git_history() -> None:
|
||||
patterns = {
|
||||
line.strip()
|
||||
for line in ROCM_DOCKERIGNORE.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip() and not line.lstrip().startswith("#")
|
||||
}
|
||||
|
||||
assert ".git" in patterns
|
||||
|
||||
|
||||
def test_rocm_source_ref_tracks_ci_vllm_release() -> None:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Integration tests: strategy overlay on the real merge_pipeline_deploy seam."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -20,6 +20,7 @@ from vllm_omni.config.composable_parallel import (
|
||||
apply_strategy_specs,
|
||||
)
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
|
||||
|
||||
@@ -137,3 +138,31 @@ def test_cli_overrides_strategy_with_warning():
|
||||
assert _resolved(stages, "talker").runtime.num_replicas == 3
|
||||
# ...and the override was warned about, naming the conflicting field.
|
||||
assert any("num_replicas" in m and "overrides the strategy-derived" in m for m in messages)
|
||||
|
||||
|
||||
def test_typed_cli_overrides_strategy():
|
||||
pipeline = OMNI_PIPELINES["qwen2_5_omni"]
|
||||
config = VllmOmniConfig.from_pipeline_config(
|
||||
pipeline,
|
||||
user_deploy_config=load_deploy_config(_DEPLOY),
|
||||
cli_overrides={"stage_1_num_replicas": 3},
|
||||
strategy_specs={"talker": [_stage_replica(2, "round_robin")]},
|
||||
)
|
||||
|
||||
talker = next(stage for stage in config.stage_configs if stage.model_stage == "talker")
|
||||
assert talker.runtime_config.num_replicas == 3
|
||||
assert config.orchestrator_config.omni_lb_policy == "round-robin"
|
||||
assert config.strategy_omni_lb_policy == "round-robin"
|
||||
|
||||
|
||||
def test_typed_tp_only_strategy_has_no_derived_lb_policy():
|
||||
pipeline = OMNI_PIPELINES["qwen2_5_omni"]
|
||||
config = VllmOmniConfig.from_pipeline_config(
|
||||
pipeline,
|
||||
user_deploy_config=load_deploy_config(_DEPLOY),
|
||||
cli_overrides={"omni_lb_policy": "round-robin"},
|
||||
strategy_specs={"thinker": [_tp(1)]},
|
||||
)
|
||||
|
||||
assert config.orchestrator_config.omni_lb_policy == "round-robin"
|
||||
assert config.strategy_omni_lb_policy is None
|
||||
@@ -1429,6 +1429,9 @@ stages:
|
||||
assert deploy.async_chunk is True
|
||||
assert deploy.connectors is not None
|
||||
assert deploy.platforms is not None
|
||||
connector = deploy.connectors["connector_of_shared_memory"]
|
||||
assert "async_chunk_batch_min_size" not in connector["extra"]
|
||||
assert "async_chunk_batch_max_wait_ms" not in connector["extra"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("hf_config", "model"),
|
||||
@@ -2625,6 +2628,15 @@ class TestPlatformOverrides:
|
||||
|
||||
assert stages[1].yaml_engine_args["dtype"] == dtype
|
||||
|
||||
def test_moss_tts_rocm_disables_codec_cudagraph(self):
|
||||
deploy_path = Path(get_deploy_config_path("moss_tts.yaml"))
|
||||
|
||||
base = load_deploy_config(deploy_path)
|
||||
assert base.stages[1].enforce_eager is False
|
||||
|
||||
rocm = _apply_platform_overrides(base, platform="rocm")
|
||||
assert rocm.stages[1].enforce_eager is True
|
||||
|
||||
def test_higgs_audio_v3_rocm_uses_triton_attention(self):
|
||||
deploy_path = Path(get_deploy_config_path("higgs_multimodal_qwen3.yaml"))
|
||||
|
||||
@@ -2688,6 +2700,63 @@ class TestPlatformOverrides:
|
||||
# Explicit null clears the inherited single-GPU 2 GiB CUDA cap.
|
||||
assert replica_stages[1].yaml_engine_args.get("kv_cache_memory_bytes") is None
|
||||
|
||||
def test_fish_speech_npu_uses_ascend_kv_block_size(self):
|
||||
deploy_path = Path(get_deploy_config_path("fish_qwen3_omni.yaml"))
|
||||
|
||||
deploy = _apply_platform_overrides(load_deploy_config(deploy_path), platform="npu")
|
||||
|
||||
assert deploy.stages[0].engine_extras["block_size"] == 128
|
||||
|
||||
@pytest.mark.parametrize("platform", ["cuda", "npu"])
|
||||
def test_recommended_native_runner_platform_defaults(self, platform):
|
||||
filename, pipeline_key = "qwen3_tts_mrv2.yaml", "qwen3_tts"
|
||||
deploy = load_deploy_config(Path(get_deploy_config_path(filename)))
|
||||
deploy = _apply_platform_overrides(deploy, platform=platform)
|
||||
expect_v2 = platform == "cuda"
|
||||
assert deploy.model_runner == ("v2" if expect_v2 else "v1")
|
||||
|
||||
# The selection must reach the live worker-dispatch consumer, not just
|
||||
# the transport-level DeployConfig field.
|
||||
pipeline = resolve_pipeline_config(pipeline_key)
|
||||
stages = merge_pipeline_deploy(pipeline, deploy)
|
||||
assert [stage.yaml_engine_args["use_v2_model_runner"] for stage in stages] == [expect_v2] * len(stages)
|
||||
|
||||
if expect_v2 and pipeline.stages and deploy.async_chunk:
|
||||
# v2 only engages the native plane on stages declaring support.
|
||||
assert all(ps.supports_native_mrv2_data_plane for ps in pipeline.stages)
|
||||
|
||||
@pytest.mark.parametrize("runner,native", [("v1", False), ("v2", False), ("v2", True)])
|
||||
def test_mrv2_undeclared_transport_warns(self, monkeypatch, runner, native):
|
||||
from unittest.mock import Mock
|
||||
|
||||
warning = Mock()
|
||||
monkeypatch.setattr("vllm_omni.config.stage_config.logger.warning", warning)
|
||||
pipeline = PipelineConfig(
|
||||
model_type="custom_pipeline",
|
||||
stages=[
|
||||
StagePipelineConfig(
|
||||
stage_id=0, model_stage="custom_ar", final_output=True, supports_native_mrv2_data_plane=native
|
||||
)
|
||||
],
|
||||
)
|
||||
merge_pipeline_deploy(pipeline, DeployConfig(model_runner=runner))
|
||||
assert warning.call_count == int(runner == "v2" and not native)
|
||||
if warning.called:
|
||||
assert "legacy transport path" in warning.call_args.args[0]
|
||||
|
||||
def test_runner_selection_rejects_engine_extras_override(self):
|
||||
pipeline = PipelineConfig(
|
||||
model_type="runner_selection_reserved",
|
||||
stages=SINGLE_STAGE_PIPE_CFG,
|
||||
)
|
||||
for reserved in ("use_v2_model_runner", "supports_native_mrv2_data_plane"):
|
||||
deploy = DeployConfig(
|
||||
model_runner="v2",
|
||||
stages=[StageDeployConfig(stage_id=0, engine_extras={reserved: False})],
|
||||
)
|
||||
with pytest.raises(ValueError, match=f"{reserved!r} must not be set"):
|
||||
merge_pipeline_deploy(pipeline, deploy)
|
||||
|
||||
def test_npu_overrides(self):
|
||||
deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml"))
|
||||
if not deploy_path.exists():
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Tests for endpoint restrictions logic."""
|
||||
|
||||
import pytest
|
||||
@@ -44,6 +44,32 @@ def test_restricted_completions_returns_400():
|
||||
assert body["error"]["type"] == "BadRequestError"
|
||||
|
||||
|
||||
def test_restricted_image_edits_returns_400():
|
||||
"""Ensure pipelines can reject image edits independently of generation."""
|
||||
app = FastAPI()
|
||||
|
||||
@app.post("/v1/images/edits")
|
||||
async def existing_handler():
|
||||
return {"ok": True}
|
||||
|
||||
@app.post("/v1/images/generations")
|
||||
async def generations_handler():
|
||||
return {"ok": True}
|
||||
|
||||
restrictions = (EndpointRestriction(OmniServingCapability.IMAGE_EDITS, REJECTION_REASON),)
|
||||
shutdown_unsupported_routes(app, restrictions)
|
||||
|
||||
client = TestClient(app)
|
||||
resp = client.post("/v1/images/edits")
|
||||
assert resp.status_code == 400
|
||||
assert resp.json()["error"]["message"] == REJECTION_REASON
|
||||
|
||||
# The restriction is endpoint-specific and must not remove generations.
|
||||
resp = client.post("/v1/images/generations")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"ok": True}
|
||||
|
||||
|
||||
def test_unrestricted_completions_not_blocked():
|
||||
"""Ensure that if we don't shutdown any routes, the route stays."""
|
||||
app = _make_app_with_server_route()
|
||||
|
||||
@@ -164,11 +164,11 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
"""Make an inventory expansion an explicit review decision."""
|
||||
category_counts = Counter(item.category for item in ENVIRONMENT_VARIABLE_INVENTORY.values())
|
||||
assert category_counts == {
|
||||
EnvironmentVariableCategory.PUBLIC_OMNI: 28,
|
||||
EnvironmentVariableCategory.PUBLIC_OMNI: 29,
|
||||
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 64,
|
||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
|
||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
|
||||
EnvironmentVariableCategory.INTERNAL: 2,
|
||||
}
|
||||
|
||||
@@ -178,11 +178,11 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
if item.category is EnvironmentVariableCategory.MODEL_SPECIFIC
|
||||
)
|
||||
assert {disposition: disposition_counts[disposition] for disposition in ModelEnvironmentVariableDisposition} == {
|
||||
ModelEnvironmentVariableDisposition.PROMOTE: 38,
|
||||
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
|
||||
ModelEnvironmentVariableDisposition.PROMOTE: 31,
|
||||
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 5,
|
||||
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
|
||||
ModelEnvironmentVariableDisposition.INTERNALIZE: 15,
|
||||
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
|
||||
ModelEnvironmentVariableDisposition.INTERNALIZE: 14,
|
||||
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 7,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,11 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
from vllm.pooling_params import PoolingParams
|
||||
|
||||
import vllm_omni.config.pipeline_registry # noqa: F401 (populate registry)
|
||||
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES as _PIPELINE_REGISTRY
|
||||
from vllm_omni.config.stage_config import (
|
||||
DeployConfig,
|
||||
StageExecutionType,
|
||||
)
|
||||
from vllm_omni.engine.stage_init_utils import (
|
||||
build_engine_args_dict_from_omni_stage_config,
|
||||
extract_stage_metadata_from_omni_stage_config,
|
||||
)
|
||||
from vllm_omni.model_executor.stage_input_processors.forced_aligner import (
|
||||
POOLING_OUTPUT_DECODER_PATH,
|
||||
TIMESTAMPS_MODALITY,
|
||||
@@ -69,3 +78,55 @@ def test_inject_noop_without_forced_aligner():
|
||||
|
||||
assert ext_pipeline is pipeline
|
||||
assert len(ext_deploy.stages) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_chunk", [True, False])
|
||||
@pytest.mark.parametrize("cli_async_chunk", [None, True, False])
|
||||
def test_aligner_uses_completed_audio_without_disabling_upstream_chunks(async_chunk, cli_async_chunk):
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
|
||||
pipeline = _PIPELINE_REGISTRY["qwen3_tts"]
|
||||
deploy = DeployConfig(async_chunk=async_chunk)
|
||||
resolution = StageConfigFactory._resolve_legacy_from_registry(
|
||||
pipeline,
|
||||
{"forced_aligner": "/models/Qwen3-ForcedAligner-0.6B", "async_chunk": cli_async_chunk},
|
||||
user_deploy_config=deploy,
|
||||
)
|
||||
stages = [stage.to_omegaconf() for stage in resolution.stage_configs]
|
||||
expected_async = async_chunk if cli_async_chunk is None else cli_async_chunk
|
||||
assert len(stages) == 3
|
||||
assert [stage.engine_args.async_chunk for stage in stages] == [expected_async, expected_async, False]
|
||||
assert stages[2].custom_process_input_func.endswith("code2wav2aligner")
|
||||
assert stages[2].engine_args.runner == "pooling"
|
||||
assert len(pipeline.stages) == 2
|
||||
assert deploy.stages == []
|
||||
|
||||
|
||||
def test_injected_aligner_survives_typed_config_and_runtime_projections():
|
||||
pipeline, deploy = inject_forced_aligner_stage(
|
||||
_PIPELINE_REGISTRY["qwen3_tts"],
|
||||
DeployConfig(),
|
||||
{"forced_aligner": "/models/Qwen3-ForcedAligner-0.6B"},
|
||||
)
|
||||
config = VllmOmniConfig.from_pipeline_config(
|
||||
pipeline,
|
||||
user_deploy_config=deploy,
|
||||
cli_overrides={"model": "/models/Qwen3-TTS"},
|
||||
)
|
||||
stage = config.stage_configs[-1]
|
||||
|
||||
assert stage.pooling_config.runner == "pooling"
|
||||
assert stage.pooling_config.pooling_output_decoder == POOLING_OUTPUT_DECODER_PATH
|
||||
assert isinstance(stage.pooling_config.default_pooling_params, PoolingParams)
|
||||
assert stage.pooling_config.default_pooling_params.task == deploy.stages[-1].default_pooling_params["task"]
|
||||
|
||||
metadata = extract_stage_metadata_from_omni_stage_config(stage)
|
||||
assert isinstance(metadata.default_sampling_params, PoolingParams)
|
||||
assert metadata.default_sampling_params.task == deploy.stages[-1].default_pooling_params["task"]
|
||||
|
||||
engine_args = build_engine_args_dict_from_omni_stage_config(
|
||||
stage,
|
||||
stage.model_config.model or "/models/Qwen3-ForcedAligner-0.6B",
|
||||
)
|
||||
assert engine_args["runner"] == "pooling"
|
||||
assert engine_args["pooling_output_decoder"] == POOLING_OUTPUT_DECODER_PATH
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.config.endpoint_policy import OmniServingCapability
|
||||
from vllm_omni.config.stage_config import StageExecutionType, load_deploy_config
|
||||
from vllm_omni.diffusion.registry import _DIFFUSION_MODELS
|
||||
from vllm_omni.model_executor.models.mammoth_moda2.pipeline import (
|
||||
MAMMOTH_MODA2_AR_PIPELINE,
|
||||
MAMMOTH_MODA2_PIPELINE,
|
||||
)
|
||||
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_mammothmoda2_generation_stage_uses_shared_diffusion_runtime() -> None:
|
||||
ar_stage, dit_stage = MAMMOTH_MODA2_PIPELINE.stages
|
||||
assert ar_stage.execution_type is StageExecutionType.LLM_AR
|
||||
assert dit_stage.execution_type is StageExecutionType.DIFFUSION
|
||||
assert dit_stage.model_arch == "MammothModa2DiTPipeline"
|
||||
assert dit_stage.custom_process_input_func.endswith(".ar2diffusion")
|
||||
assert dit_stage.omni_kv_config == {"need_recv_cache": False}
|
||||
assert dit_stage.input_sources == (0,)
|
||||
assert dit_stage.final_output is True
|
||||
assert dit_stage.final_output_type == "image"
|
||||
|
||||
|
||||
def test_mammothmoda2_dit_is_registered_only_with_diffusion_runtime() -> None:
|
||||
assert _DIFFUSION_MODELS["MammothModa2DiTPipeline"] == (
|
||||
"mammoth_moda2",
|
||||
"pipeline_mammothmoda2_dit",
|
||||
"MammothModa2DiTPipeline",
|
||||
)
|
||||
assert "MammothModa2DiTPipeline" not in _OMNI_MODELS
|
||||
|
||||
|
||||
def test_mammothmoda2_deploy_does_not_shadow_request_sampling_fields() -> None:
|
||||
deploy = load_deploy_config(get_deploy_config_path("mammoth_moda2.yaml"))
|
||||
|
||||
assert deploy.stages[1].default_sampling_params is None
|
||||
|
||||
|
||||
def test_mammothmoda2_rejects_unsupported_image_edits() -> None:
|
||||
restrictions = {
|
||||
restriction.capability: restriction.reason for restriction in MAMMOTH_MODA2_PIPELINE.endpoint_restrictions
|
||||
}
|
||||
|
||||
assert set(restrictions) == {OmniServingCapability.IMAGE_EDITS}
|
||||
assert "img2img" in restrictions[OmniServingCapability.IMAGE_EDITS]
|
||||
assert MAMMOTH_MODA2_AR_PIPELINE.endpoint_restrictions == ()
|
||||
|
||||
|
||||
def test_mammothmoda2_ar_only_topology_is_unchanged() -> None:
|
||||
assert len(MAMMOTH_MODA2_AR_PIPELINE.stages) == 1
|
||||
stage = MAMMOTH_MODA2_AR_PIPELINE.stages[0]
|
||||
assert stage.execution_type is StageExecutionType.LLM_AR
|
||||
assert stage.final_output_type == "text"
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from dataclasses import fields
|
||||
from inspect import Parameter, signature
|
||||
from multiprocessing.reduction import ForkingPickler
|
||||
@@ -11,6 +12,7 @@ from pathlib import Path
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import ValidationError
|
||||
from pydantic.fields import FieldInfo
|
||||
from transformers import Qwen3OmniMoeConfig
|
||||
@@ -23,6 +25,7 @@ from vllm.config import SchedulerConfig as VllmSchedulerConfig
|
||||
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.config import omni_config as omni_config_module
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.config.omni_config import (
|
||||
BaseVllmOmniStageConfig,
|
||||
OmniStageCacheConfig,
|
||||
@@ -47,8 +50,10 @@ from vllm_omni.config.stage_config import (
|
||||
StageDeployConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
_apply_platform_overrides,
|
||||
load_deploy_config,
|
||||
merge_pipeline_deploy,
|
||||
resolve_deploy_yaml,
|
||||
)
|
||||
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||
from vllm_omni.engine.stage_engine_startup import _serialize_stage_config
|
||||
@@ -59,6 +64,24 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
_DEPLOY_DIR = Path(__file__).parents[2] / "vllm_omni" / "deploy"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_chunk", [False, True])
|
||||
def test_native_kv_transfer_requires_completed_ar_stage(async_chunk):
|
||||
from types import SimpleNamespace
|
||||
|
||||
pipeline = SimpleNamespace(stages=(), model_type="test")
|
||||
deploy = DeployConfig(
|
||||
async_chunk=async_chunk,
|
||||
stages=[
|
||||
StageDeployConfig(stage_id=0, engine_extras={"kv_transfer_config": {"kv_connector": "MooncakeConnector"}})
|
||||
],
|
||||
)
|
||||
if async_chunk:
|
||||
with pytest.raises(ValueError, match="requires async_chunk=False"):
|
||||
omni_config_module._validate_async_chunk_support(pipeline, deploy)
|
||||
else:
|
||||
omni_config_module._validate_async_chunk_support(pipeline, deploy)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _stable_test_platform(monkeypatch):
|
||||
from vllm_omni import platforms
|
||||
@@ -94,6 +117,20 @@ def _from_pipeline_key(
|
||||
)
|
||||
|
||||
|
||||
def test_mammothmoda2_diffusion_stage_projects_native_backend_config() -> None:
|
||||
config = _from_pipeline_key(
|
||||
"mammoth_moda2",
|
||||
cli_overrides={"model": "/models/MammothModa2-Preview"},
|
||||
)
|
||||
stage = config.stage_by_id(1)
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.diffusion_config.model_class_name == "MammothModa2DiTPipeline"
|
||||
assert stage.diffusion_config.model == "/models/MammothModa2-Preview"
|
||||
assert stage.diffusion_config.step_execution is False
|
||||
assert stage.scheduler_config.max_num_seqs == 1
|
||||
assert stage.connector_config.omni_kv_config == {"need_recv_cache": False}
|
||||
|
||||
|
||||
def test_minimax_h3_text_encoder_tp_targets_only_structured_stage_zero() -> None:
|
||||
config = _from_pipeline_key(
|
||||
"minimax_h3_disaggregated",
|
||||
@@ -123,6 +160,42 @@ def test_non_duplex_deploy_keeps_model_session_capacity_at_one(tmp_path: Path) -
|
||||
assert [stage.model_config.session_mode for stage in omni_config.stage_configs] == ["turn", "turn"]
|
||||
|
||||
|
||||
def test_nested_stage_override_deep_merges_structured_model_config() -> None:
|
||||
config = _from_pipeline_key(
|
||||
"cosmos3_policy",
|
||||
deploy_config_path=get_deploy_config_path("cosmos3_policy_droid.yaml"),
|
||||
cli_overrides={"stage_0_model_config": {"guardrails": False}},
|
||||
)
|
||||
|
||||
model_config = config.stage_by_id(0).diffusion_config.model_config
|
||||
assert model_config["guardrails"] is False
|
||||
assert model_config["policy_server_config"]["action_space"] == "joint_position"
|
||||
|
||||
|
||||
def _build_single_diffusion_config(
|
||||
*,
|
||||
cli_overrides: dict | None = None,
|
||||
engine_extras: dict | None = None,
|
||||
) -> VllmOmniConfig:
|
||||
pipeline = PipelineConfig(
|
||||
model_type="diffusion-owner-test",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="diffusion",
|
||||
execution_type=StageExecutionType.DIFFUSION,
|
||||
final_output=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
deploy_stages = [StageDeployConfig(stage_id=0, engine_extras=engine_extras)] if engine_extras is not None else []
|
||||
return VllmOmniConfig.from_pipeline_config(
|
||||
pipeline,
|
||||
user_deploy_config=DeployConfig(async_chunk=False, stages=deploy_stages),
|
||||
cli_overrides=cli_overrides,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", sorted(OMNI_PIPELINES))
|
||||
def test_vllm_omni_config_from_pipeline_config_matches_merge_pipeline_deploy(model_type: str):
|
||||
pipeline = _resolve_pipeline_or_skip(model_type)
|
||||
@@ -219,7 +292,7 @@ def test_from_pipeline_config_normalizes_stage_engine_extras_without_expanding_s
|
||||
def test_frontend_log_stats_flag_is_not_an_unowned_stage_argument(disabled):
|
||||
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
|
||||
|
||||
config = _from_pipeline_key("dots_tts", cli_overrides={"disable_log_stats": disabled})
|
||||
config = _from_pipeline_key("voxcpm2", cli_overrides={"disable_log_stats": disabled})
|
||||
assert config.stage_configs
|
||||
engine_args = build_engine_args_dict_from_omni_stage_config(config.stage_by_id(0), model="test-model")
|
||||
assert "disable_log_stats" not in engine_args
|
||||
@@ -242,6 +315,30 @@ def test_from_pipeline_config_applies_cli_overrides_without_stage_config_runtime
|
||||
assert stage1.runtime_config.num_gpus == stage1.parallel_config.world_size
|
||||
|
||||
|
||||
def test_diffusion_cli_parallel_overrides_beat_nested_deploy_parallel_config():
|
||||
"""Flat CLI parallel flags override the deploy YAML's nested parallel_config.
|
||||
|
||||
Regression for the NPU nightly failure (#7778): hunyuan_image3_dit.yaml's
|
||||
platform section sets nested parallel_config.tensor_parallel_size=4 while
|
||||
the perf tests pass --tensor-parallel-size 2. The flat CLI value must win,
|
||||
mirroring StageConfig.to_omegaconf, or the stage demands more devices than
|
||||
the machine has (tp=4 x usp=2 = 8 on a 4-card box).
|
||||
"""
|
||||
omni_config = VllmOmniConfig.from_pipeline_config(
|
||||
_resolve_pipeline_or_skip("hunyuan_image3_dit"),
|
||||
user_deploy_config=DeployConfig(
|
||||
stages=[StageDeployConfig(stage_id=0, engine_extras={"parallel_config": {"tensor_parallel_size": 4}})]
|
||||
),
|
||||
cli_overrides={"tensor_parallel_size": 2, "ulysses_degree": 2},
|
||||
)
|
||||
|
||||
stage = omni_config.stage_by_id(0)
|
||||
|
||||
assert stage.parallel_config.tensor_parallel_size == 2
|
||||
assert stage.parallel_config.ulysses_degree == 2
|
||||
assert stage.parallel_config.world_size == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cli_overrides",
|
||||
[
|
||||
@@ -424,7 +521,7 @@ def test_from_pipeline_config_normalizes_nested_parallel_config_aliases(field_na
|
||||
assert "data_parallel_master_ip" in stage.parallel_config._omni_explicit_fields
|
||||
|
||||
|
||||
def test_from_pipeline_config_accepts_diffusion_only_cli_fields_for_diffusion_stage():
|
||||
def test_from_pipeline_config_keeps_global_kv_cache_dtype_outside_diffusion_stage():
|
||||
stage = _from_pipeline_key(
|
||||
"dreamzero",
|
||||
deploy_config_path="dreamzero_tp1_cfg2",
|
||||
@@ -432,11 +529,57 @@ def test_from_pipeline_config_accepts_diffusion_only_cli_fields_for_diffusion_st
|
||||
).stage_by_id(0)
|
||||
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.diffusion_config.diffusion_kv_cache_dtype == "fp8"
|
||||
assert stage.diffusion_config.diffusion_kv_cache_dtype is None
|
||||
|
||||
|
||||
def test_stage_cli_field_selection_defers_ownership_validation_until_sources_are_merged():
|
||||
assert omni_config_module._stage_cli_overrides(0, {"enable_lora": True}) == {"enable_lora": True}
|
||||
assert omni_config_module._stage_cli_overrides(
|
||||
0,
|
||||
{"enable_lora": True},
|
||||
) == {"enable_lora": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cli_overrides", "expected_dtype"),
|
||||
[
|
||||
({}, torch.float32),
|
||||
({"dtype": None}, torch.float32),
|
||||
({"dtype": "float16"}, torch.float16),
|
||||
({"dtype": "auto"}, torch.bfloat16),
|
||||
({"stage_0_dtype": "bfloat16"}, torch.bfloat16),
|
||||
],
|
||||
)
|
||||
def test_diffusion_deploy_dtype_survives_unset_cli_overrides(cli_overrides, expected_dtype):
|
||||
stage = _build_single_diffusion_config(
|
||||
engine_extras={"dtype": "float32", "cache_backend": "tea_cache"},
|
||||
cli_overrides=cli_overrides,
|
||||
).stage_by_id(0)
|
||||
|
||||
assert stage.diffusion_config.dtype == expected_dtype
|
||||
assert stage.diffusion_config.cache_backend == "tea_cache"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"coordination_kwargs",
|
||||
[
|
||||
{"omni_master_address": "127.0.0.1", "omni_master_port": 30000},
|
||||
{"omni_dp_size_local": 2, "omni_heartbeat_timeout": 60.0, "omni_lb_policy": "random"},
|
||||
],
|
||||
)
|
||||
def test_diffusion_ingress_routes_coordination_fields_away(coordination_kwargs):
|
||||
normalize = omni_config_module.normalize_and_validate_diffusion_engine_ingress_kwargs
|
||||
assert normalize({**coordination_kwargs, "enable_sleep_mode": True}, stage_id=0) == {"enable_sleep_mode": True}
|
||||
with pytest.raises(ValueError, match="omni_master_unknown_field"):
|
||||
normalize({**coordination_kwargs, "omni_master_unknown_field": "unknown"}, stage_id=0)
|
||||
|
||||
|
||||
def test_diffusion_ingress_defers_defaults(monkeypatch):
|
||||
monkeypatch.setenv("DIFFUSION_CACHE_BACKEND", "tea_cache")
|
||||
|
||||
assert omni_config_module.normalize_and_validate_diffusion_engine_ingress_kwargs({}, stage_id=0) == {}
|
||||
assert omni_config_module.normalize_and_validate_diffusion_engine_ingress_kwargs(
|
||||
{"dtype": None, "cache_backend": None}, stage_id=0
|
||||
) == {"dtype": None, "cache_backend": None}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -642,6 +785,67 @@ def test_joyai_code2wav_waits_for_full_payload():
|
||||
assert code2wav.model_config.requires_full_payload_input is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_runner", ["v1", "v2"])
|
||||
def test_deploy_model_runner_selection_propagates_to_every_stage(tmp_path: Path, model_runner: str):
|
||||
deploy_path = tmp_path / "qwen3_tts_runner.yaml"
|
||||
deploy_path.write_text(
|
||||
f"""\
|
||||
pipeline: qwen3_tts
|
||||
model_runner: {model_runner}
|
||||
async_chunk: true
|
||||
stages:
|
||||
- stage_id: 0
|
||||
- stage_id: 1
|
||||
"""
|
||||
)
|
||||
|
||||
deploy = load_deploy_config(deploy_path)
|
||||
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
|
||||
legacy_stages = merge_pipeline_deploy(pipeline, deploy)
|
||||
structured = _from_pipeline_key("qwen3_tts", deploy_config_path=str(deploy_path))
|
||||
expect_v2 = model_runner == "v2"
|
||||
|
||||
assert deploy.model_runner == model_runner
|
||||
assert all(stage.yaml_engine_args["use_v2_model_runner"] is expect_v2 for stage in legacy_stages)
|
||||
assert all(stage.model_config.use_v2_model_runner is expect_v2 for stage in structured.stage_configs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("platform", ["npu", "xpu"])
|
||||
def test_mrv2_fails_fast_on_platforms_without_native_workers(platform: str):
|
||||
with pytest.raises(NotImplementedError, match="Model Runner V2"):
|
||||
_apply_platform_overrides(DeployConfig(model_runner="v2"), platform=platform)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("default_name", "mrv2_name"),
|
||||
[
|
||||
("qwen3_tts.yaml", "qwen3_tts_mrv2.yaml"),
|
||||
(
|
||||
"qwen3_tts_high_concurrency.yaml",
|
||||
"qwen3_tts_high_concurrency_mrv2.yaml",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_qwen3_mrv2_profiles_are_explicit_opt_in(default_name: str, mrv2_name: str):
|
||||
assert load_deploy_config(_DEPLOY_DIR / default_name).model_runner == "v1"
|
||||
assert load_deploy_config(_DEPLOY_DIR / mrv2_name).model_runner == "v2"
|
||||
|
||||
|
||||
def test_qwen3_tts_mrv2_retunes_do_not_change_default_mrv1_profile():
|
||||
default = resolve_deploy_yaml(_DEPLOY_DIR / "qwen3_tts_high_concurrency.yaml")
|
||||
mrv2 = resolve_deploy_yaml(_DEPLOY_DIR / "qwen3_tts_high_concurrency_mrv2.yaml")
|
||||
default_extra = default["connectors"]["connector_of_shared_memory"]["extra"]
|
||||
mrv2_extra = mrv2["connectors"]["connector_of_shared_memory"]["extra"]
|
||||
|
||||
# B2 batch stays consistent with the graph buckets; V1 profile untouched.
|
||||
assert default_extra["decode_batch_max_size"] == 1
|
||||
assert mrv2_extra["decode_batch_max_size"] == 2
|
||||
assert mrv2_extra["decode_cudagraph_batch_sizes"] == [1, 2]
|
||||
assert mrv2_extra["code_predictor_prefix_graphs"] is False
|
||||
assert mrv2["stages"][1]["enforce_eager"] is False
|
||||
assert default["stages"][1]["enforce_eager"] is True
|
||||
|
||||
|
||||
def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
|
||||
assert not hasattr(BaseVllmOmniStageConfig, "from_stage_config")
|
||||
assert not hasattr(BaseVllmOmniStageConfig, "to_legacy_stage_config")
|
||||
@@ -654,6 +858,7 @@ def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
|
||||
"load_config",
|
||||
"cache_config",
|
||||
"scheduler_config",
|
||||
"pooling_config",
|
||||
"connector_config",
|
||||
"runtime_config",
|
||||
"parallel_config",
|
||||
@@ -737,9 +942,11 @@ def test_sub_config_fields_match_structured_scopes():
|
||||
"limit_mm_per_prompt",
|
||||
"interleave_mm_strings",
|
||||
"media_io_kwargs",
|
||||
"final_output",
|
||||
"active_stream_window",
|
||||
"session_mode",
|
||||
"duplex_max_sessions",
|
||||
"use_v2_model_runner",
|
||||
"enable_sleep_mode",
|
||||
"default_sampling_params",
|
||||
"subtalker_sampling_params",
|
||||
@@ -748,6 +955,7 @@ def test_sub_config_fields_match_structured_scopes():
|
||||
"custom_voice_dir",
|
||||
"task_type",
|
||||
"codec_frame_rate_hz",
|
||||
"supports_native_mrv2_data_plane",
|
||||
"enforce_eager",
|
||||
"max_cudagraph_capture_size",
|
||||
"enable_flashinfer_autotune",
|
||||
@@ -797,6 +1005,7 @@ def test_sub_config_fields_match_structured_scopes():
|
||||
assert {f.name for f in fields(OmniStageConnectorConfig)} == {
|
||||
"async_chunk",
|
||||
"omni_kv_config",
|
||||
"kv_transfer_config",
|
||||
"stage_connector",
|
||||
"output_connectors",
|
||||
"input_connectors",
|
||||
@@ -889,6 +1098,17 @@ def test_structured_llm_stage_registration_payloads_remain_msgpack_transport_saf
|
||||
msgspec.msgpack.encode(payload)
|
||||
|
||||
|
||||
def test_structured_diffusion_torch_dtype_is_msgpack_transport_safe():
|
||||
stage_config = _from_pipeline_key(
|
||||
"hunyuan_image3_dit",
|
||||
cli_overrides={"dtype": torch.bfloat16},
|
||||
).stage_by_id(0)
|
||||
|
||||
serialized = _serialize_stage_config(stage_config)
|
||||
assert serialized["diffusion_config"]["dtype"] == "bfloat16"
|
||||
msgspec.msgpack.encode(serialized)
|
||||
|
||||
|
||||
def test_diffusion_parallel_config_fields_cover_legacy_surface():
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
|
||||
@@ -1352,18 +1572,19 @@ def test_diffusion_config_from_kwargs_reuses_legacy_normalization(monkeypatch):
|
||||
monkeypatch.setenv("DIFFUSION_CACHE_BACKEND", "TEA_CACHE")
|
||||
monkeypatch.setattr(current_omni_platform, "is_cuda", lambda: True)
|
||||
|
||||
cfg = omni_config_module._DiffusionConfigProjection.from_kwargs(
|
||||
diffusion_attention_backend="flash_attn",
|
||||
fa_deterministic=True,
|
||||
kv_cache_dtype="fp8",
|
||||
kv_cache_skip_steps="0-1",
|
||||
kv_cache_skip_layers=[2],
|
||||
static_lora_scale=0.25,
|
||||
diffusion_kv_mode="paged_scheduler",
|
||||
diffusion_kv_max_rows_per_request=2,
|
||||
diffusers_load_kwargs=None,
|
||||
diffusers_call_kwargs=None,
|
||||
)
|
||||
with pytest.warns(FutureWarning):
|
||||
cfg = omni_config_module._DiffusionConfigProjection.from_kwargs(
|
||||
diffusion_attention_backend="flash_attn",
|
||||
fa_deterministic=True,
|
||||
kv_cache_dtype="fp8",
|
||||
kv_cache_skip_steps="0-1",
|
||||
kv_cache_skip_layers=[2],
|
||||
static_lora_scale=0.25,
|
||||
diffusion_kv_mode="paged_scheduler",
|
||||
diffusion_kv_max_rows_per_request=2,
|
||||
diffusers_load_kwargs=None,
|
||||
diffusers_call_kwargs=None,
|
||||
)
|
||||
|
||||
assert cfg.diffusion_attention_config.default.backend == "flash_attn"
|
||||
assert cfg.fa_deterministic is True
|
||||
@@ -1400,47 +1621,6 @@ def test_diffusion_config_none_values_preserve_dataclass_defaults():
|
||||
assert config.enable_sleep_mode is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("canonical_key", "alias_key", "canonical_value", "alias_value"),
|
||||
[
|
||||
("lora_scale", "static_lora_scale", 0.75, 0.25),
|
||||
(
|
||||
"quantization_config",
|
||||
"diffusion_quantization_config",
|
||||
{"method": "canonical"},
|
||||
{"method": "diffusion-alias"},
|
||||
),
|
||||
(
|
||||
"quantization_config",
|
||||
"quantization",
|
||||
{"method": "canonical"},
|
||||
"legacy-alias",
|
||||
),
|
||||
("diffusion_kv_cache_dtype", "kv_cache_dtype", "fp8", "fp16"),
|
||||
("diffusion_kv_cache_skip_steps", "kv_cache_skip_steps", "0-1", "2-3"),
|
||||
("diffusion_kv_cache_skip_layers", "kv_cache_skip_layers", "1-2", "3-4"),
|
||||
("streaming_output", "diffusion_streaming_output", False, True),
|
||||
],
|
||||
)
|
||||
def test_diffusion_alias_conflicts_prefer_canonical_key(
|
||||
canonical_key,
|
||||
alias_key,
|
||||
canonical_value,
|
||||
alias_value,
|
||||
):
|
||||
from vllm_omni.diffusion.data import normalize_omni_diffusion_kwargs
|
||||
|
||||
normalized = normalize_omni_diffusion_kwargs(
|
||||
{
|
||||
canonical_key: canonical_value,
|
||||
alias_key: alias_value,
|
||||
}
|
||||
)
|
||||
|
||||
assert normalized[canonical_key] == canonical_value
|
||||
assert alias_key not in normalized
|
||||
|
||||
|
||||
def test_from_pipeline_config_normalizes_diffusion_config_aliases_from_engine_args(tmp_path, monkeypatch):
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -1465,10 +1645,11 @@ def test_from_pipeline_config_normalizes_diffusion_config_aliases_from_engine_ar
|
||||
)
|
||||
)
|
||||
|
||||
stage = _from_pipeline_key(
|
||||
"dreamzero",
|
||||
deploy_config_path=str(deploy_path),
|
||||
).stage_by_id(0)
|
||||
with pytest.warns(FutureWarning):
|
||||
stage = _from_pipeline_key(
|
||||
"dreamzero",
|
||||
deploy_config_path=str(deploy_path),
|
||||
).stage_by_id(0)
|
||||
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.diffusion_config.diffusion_attention_config.default.backend == "flash_attn"
|
||||
@@ -1480,8 +1661,11 @@ def test_from_pipeline_config_normalizes_diffusion_config_aliases_from_engine_ar
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
|
||||
|
||||
engine_args = build_engine_args_dict_from_omni_stage_config(stage, model="test-model")
|
||||
od_config = OmniDiffusionConfig.from_kwargs(**engine_args)
|
||||
engine_args = build_engine_args_dict_from_omni_stage_config(stage, model=str(tmp_path))
|
||||
diffusion_kwargs = omni_config_module.extract_diffusion_stage_config_kwargs(
|
||||
engine_args, stage_id=stage.stage_id, include_engine_adapter_metadata=True
|
||||
)
|
||||
od_config = OmniDiffusionConfig.from_kwargs(**diffusion_kwargs)
|
||||
assert od_config.kv_transfer_config.engine_id == "dit-engine-1"
|
||||
|
||||
|
||||
@@ -1634,10 +1818,8 @@ def test_video_output_transport_survives_stage_override_filtering() -> None:
|
||||
|
||||
|
||||
def test_video_output_transport_reaches_default_diffusion_stage() -> None:
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
|
||||
transport = {"enable_device_postprocess": True}
|
||||
stages = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
||||
stages = StageConfigFactory.create_default_diffusion(
|
||||
{
|
||||
"model": "unused",
|
||||
"model_class_name": "UnknownPipeline",
|
||||
@@ -1705,3 +1887,157 @@ def test_compact_offload_config_is_validated_during_projection():
|
||||
"components": ["dit"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pipeline_async", [True, False])
|
||||
@pytest.mark.parametrize("stage_async", [None, False, True])
|
||||
def test_stage_async_chunk_opt_out_matches_legacy_config(pipeline_async, stage_async):
|
||||
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
|
||||
deploy = DeployConfig(
|
||||
async_chunk=pipeline_async,
|
||||
stages=[StageDeployConfig(stage_id=i, async_chunk=stage_async) for i in (0, 1)],
|
||||
)
|
||||
config = VllmOmniConfig.from_pipeline_config(pipeline, user_deploy_config=deploy)
|
||||
legacy = merge_pipeline_deploy(pipeline, deploy)
|
||||
expected = pipeline_async and stage_async is not False
|
||||
assert config.stage_by_id(0).connector_config.async_chunk is expected
|
||||
assert config.stage_by_id(1).connector_config.async_chunk is expected
|
||||
assert legacy[1].yaml_engine_args["async_chunk"] is expected
|
||||
assert config.stage_by_id(1).custom_process_input_func == legacy[1].custom_process_input_func
|
||||
if not expected:
|
||||
assert config.stage_by_id(1).custom_process_input_func.endswith("talker2code2wav_token_only")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled_stage", [0, 1])
|
||||
@pytest.mark.parametrize("builder", [merge_pipeline_deploy, VllmOmniConfig.from_pipeline_config])
|
||||
def test_async_chunk_rejects_mismatched_connector_edge(disabled_stage, builder):
|
||||
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
|
||||
deploy = DeployConfig(
|
||||
async_chunk=True,
|
||||
stages=[StageDeployConfig(stage_id=disabled_stage, async_chunk=False)],
|
||||
)
|
||||
with pytest.raises(ValueError, match="incompatible async_chunk settings on connector edge 0 -> 1"):
|
||||
if builder is merge_pipeline_deploy:
|
||||
builder(pipeline, deploy)
|
||||
else:
|
||||
builder(pipeline, user_deploy_config=deploy)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("explicit", [False, True])
|
||||
def test_diffusion_quantization_origin_survives_projection_and_transport(monkeypatch, explicit):
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
|
||||
from vllm_omni.quantization import build_quant_config
|
||||
|
||||
checkpoint = TransformerConfig.from_dict(
|
||||
{
|
||||
"quantization_config": {
|
||||
"quant_method": "mxfp4",
|
||||
"is_checkpoint_mxfp4_serialized": True,
|
||||
"w4a8_fallback_steps": [37],
|
||||
}
|
||||
}
|
||||
)
|
||||
requested = build_quant_config("mxfp4", w4a8_fallback_steps=[]) if explicit else None
|
||||
cfg = omni_config_module._DiffusionConfigProjection.from_kwargs(
|
||||
tf_model_config=checkpoint,
|
||||
quantization_config=requested,
|
||||
)
|
||||
assert cfg.quantization_config_is_auto_detected is not explicit
|
||||
monkeypatch.setattr(OmniDiffusionConfig, "_resolve_master_port", lambda _self: 29500)
|
||||
monkeypatch.setattr(OmniDiffusionConfig, "enrich_config", lambda self: self.set_tf_model_config(checkpoint))
|
||||
cfg.enrich_config()
|
||||
restored = ForkingPickler.loads(ForkingPickler.dumps(cfg))
|
||||
assert restored.quantization_config_is_auto_detected is not explicit
|
||||
assert restored.quantization_config.w4a8_fallback_steps == ([] if explicit else [37])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"factory",
|
||||
[
|
||||
omni_config_module._DiffusionConfigProjection.from_kwargs,
|
||||
pytest.param(
|
||||
lambda **kwargs: __import__(
|
||||
"vllm_omni.diffusion.data",
|
||||
fromlist=["OmniDiffusionConfig"],
|
||||
).OmniDiffusionConfig.from_kwargs(**kwargs),
|
||||
id="runtime-config",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_diffusion_config_entrypoints_reject_unknown_none(factory):
|
||||
with pytest.raises((ValueError, ValidationError), match="enable_sleep_mod"):
|
||||
factory(enable_sleep_mod=None)
|
||||
|
||||
|
||||
def test_diffusion_config_projection_rejects_unknown_none():
|
||||
with pytest.raises(ValueError, match="enable_sleep_mod"):
|
||||
omni_config_module._DiffusionConfigProjection.from_kwargs(enable_sleep_mod=None)
|
||||
|
||||
|
||||
def test_runtime_diffusion_config_rejects_unknown_none():
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
with pytest.raises(ValueError, match="enable_sleep_mod"):
|
||||
OmniDiffusionConfig.from_kwargs(enable_sleep_mod=None)
|
||||
|
||||
|
||||
def test_direct_diffusion_aliases_promote_none_and_reject_real_conflicts(monkeypatch):
|
||||
from vllm_omni.diffusion import data as diffusion_data
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
monkeypatch.setattr(diffusion_data, "build_quant_config", lambda config: config)
|
||||
with pytest.warns(FutureWarning) as warnings:
|
||||
config = OmniDiffusionConfig.from_kwargs(
|
||||
quantization={"method": "example"},
|
||||
quantization_config=None,
|
||||
kv_cache_dtype="fp8",
|
||||
diffusion_kv_cache_dtype=None,
|
||||
enable_sleep_mode=False,
|
||||
request_batch_max_wait_ms=0.0,
|
||||
)
|
||||
|
||||
warning_messages = [str(warning.message) for warning in warnings]
|
||||
assert any("quantization" in message for message in warning_messages)
|
||||
assert any("kv_cache_dtype" in message for message in warning_messages)
|
||||
assert config.quantization_config == {"method": "example"}
|
||||
assert config.diffusion_kv_cache_dtype == "fp8"
|
||||
assert config.enable_sleep_mode is False
|
||||
assert config.request_batch_max_wait_ms == 0.0
|
||||
with pytest.raises(ValueError, match=r"quantization.*quantization_config"):
|
||||
OmniDiffusionConfig.from_kwargs(quantization="fp8", quantization_config={"method": "example"})
|
||||
|
||||
|
||||
def test_structured_diffusion_stage_rejects_unknown_engine_extra():
|
||||
with pytest.raises(ValueError, match=r"stage 0.*enable_sleep_mod"):
|
||||
_build_single_diffusion_config(engine_extras={"enable_sleep_mod": None})
|
||||
|
||||
with pytest.raises(ValueError, match=r"stage 0.*enable_sleep_mod"):
|
||||
_build_single_diffusion_config(cli_overrides={"enable_sleep_mod": None})
|
||||
|
||||
with pytest.raises(ValueError, match=r"stage 0.*enable_sleep_mod"):
|
||||
_build_single_diffusion_config(cli_overrides={"stage_0_enable_sleep_mod": None})
|
||||
|
||||
with pytest.raises(ValueError, match=r"no structured config owner: parallel_config\.unknown_field"):
|
||||
_build_single_diffusion_config(engine_extras={"parallel_config": {"unknown_field": 1}})
|
||||
|
||||
|
||||
def test_structured_diffusion_stage_keeps_shared_globals_outside_diffusion():
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
stage = _build_single_diffusion_config(
|
||||
cli_overrides={"kv_cache_dtype": "fp8", "seed": 7},
|
||||
).stage_by_id(0)
|
||||
|
||||
assert stage.diffusion_config.diffusion_kv_cache_dtype is None
|
||||
assert not any(issubclass(warning.category, FutureWarning) for warning in caught)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field_name", "config_kwargs"),
|
||||
[
|
||||
("seed", {"cli_overrides": {"stage_0_seed": None}}),
|
||||
("kv_cache_dtype", {"engine_extras": {"kv_cache_dtype": "fp8"}}),
|
||||
],
|
||||
)
|
||||
def test_structured_diffusion_stage_rejects_explicit_shared_engine_field(field_name, config_kwargs):
|
||||
with pytest.raises(ValueError, match=rf"stage 0.*{field_name}"):
|
||||
_build_single_diffusion_config(**config_kwargs)
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Check the sleep/wake E2E topology before loading large BAGEL weights."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.test_omni_sleep_mode import _sleep_deploy_config
|
||||
from vllm_omni.config.omni_config import (
|
||||
VllmOmniConfig,
|
||||
VllmOmniDiffusionStageConfig,
|
||||
extract_diffusion_stage_config_kwargs,
|
||||
)
|
||||
from vllm_omni.config.pipeline_registry import resolve_pipeline_config
|
||||
from vllm_omni.config.stage_config import load_deploy_config
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("diffusion_only,tp_size", [(False, 1), (False, 2), (True, 2)])
|
||||
def test_sleep_deploy_resolves_expected_stages(diffusion_only, tp_size):
|
||||
deploy = load_deploy_config(_sleep_deploy_config(diffusion_only=diffusion_only, tp_size=tp_size))
|
||||
pipeline = resolve_pipeline_config(deploy.pipeline or "bagel")
|
||||
config = VllmOmniConfig.from_pipeline_config(pipeline, user_deploy_config=deploy)
|
||||
|
||||
assert len(config.stage_configs) == (1 if diffusion_only else 2)
|
||||
for stage in config.stage_configs:
|
||||
is_diffusion = diffusion_only or stage.stage_id == 1
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig) == is_diffusion
|
||||
# Keep BAGEL's batch budget: a single multimodal item needs 8625 tokens.
|
||||
assert stage.scheduler_config.max_num_batched_tokens == 32768
|
||||
assert stage.model_config.enable_sleep_mode is True
|
||||
assert stage.parallel_config.tensor_parallel_size == tp_size
|
||||
assert stage.runtime_config.devices == ",".join(str(stage.stage_id * tp_size + rank) for rank in range(tp_size))
|
||||
if not is_diffusion:
|
||||
assert stage.cache_config.gpu_memory_utilization == 0.8
|
||||
if is_diffusion:
|
||||
engine_args = build_engine_args_dict_from_omni_stage_config(stage, "test-bagel")
|
||||
terminal = OmniDiffusionConfig.from_kwargs(
|
||||
**extract_diffusion_stage_config_kwargs(
|
||||
engine_args, stage_id=stage.stage_id, include_engine_adapter_metadata=True
|
||||
)
|
||||
)
|
||||
assert terminal.enable_sleep_mode is True
|
||||
assert terminal.parallel_config.tensor_parallel_size == tp_size
|
||||
|
||||
if not diffusion_only:
|
||||
assert config.stage_by_id(1).stage_pipeline_config.input_sources == (0,)
|
||||
assert deploy.stages[1].input_connectors["from_stage_0"] == "shared_memory_connector"
|
||||
@@ -1,14 +1,22 @@
|
||||
"""Test that OmniGenerationScheduler restores chunk-waiting requests
|
||||
even when the OmniNewRequestData rewrapping fails.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
Regression test: if process_pending_chunks() moves requests into
|
||||
internal deques but restore_queues() is not called due to an exception,
|
||||
those requests are permanently orphaned.
|
||||
"""
|
||||
"""Chunk-lifecycle coverage for OmniGenerationScheduler: restore on error,
|
||||
native terminal state, in-flight no-resubmission, slot release on finish,
|
||||
and single scheduling of a terminal empty-prompt chunk."""
|
||||
|
||||
from collections import deque
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm import SamplingParams
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler, _has_async_chunk_payload_to_run
|
||||
from vllm_omni.engine.serialization import serialize_additional_information
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -20,16 +28,20 @@ class FakeAdapter:
|
||||
self.waiting_for_chunk_waiting_requests = deque()
|
||||
self.waiting_for_chunk_running_requests = deque()
|
||||
self.restore_called = False
|
||||
self.done_request_ids = set()
|
||||
|
||||
def process_pending_chunks(self, waiting, running):
|
||||
"""Simulate moving requests out of the scheduler queues."""
|
||||
# Move one request from running into internal deque
|
||||
def process_pending_chunks(self, waiting, running, scheduler_requests=None):
|
||||
if running:
|
||||
req = running.pop()
|
||||
self.waiting_for_chunk_running_requests.append(req)
|
||||
|
||||
def is_done_receiving_chunks(self, request_id):
|
||||
return request_id in self.done_request_ids
|
||||
|
||||
def collect_failed_send_request_ids(self):
|
||||
return {}
|
||||
|
||||
def restore_queues(self, waiting, running, scheduler_requests=None):
|
||||
"""Put requests back."""
|
||||
self.restore_called = True
|
||||
running.extend(self.waiting_for_chunk_running_requests)
|
||||
self.waiting_for_chunk_running_requests = deque()
|
||||
@@ -38,6 +50,131 @@ class FakeAdapter:
|
||||
pass
|
||||
|
||||
|
||||
def _make_generation_scheduler(waiting_request, *, use_v2_model_runner=False):
|
||||
scheduler = OmniGenerationScheduler.__new__(OmniGenerationScheduler)
|
||||
scheduler.max_num_scheduled_tokens = 8
|
||||
scheduler.max_num_running_reqs = 1
|
||||
scheduler._pause_state = PauseState.UNPAUSED
|
||||
scheduler.running = []
|
||||
scheduler.waiting = create_request_queue(SchedulingPolicy.FCFS)
|
||||
scheduler.waiting.add_request(waiting_request)
|
||||
scheduler.skipped_waiting = create_request_queue(SchedulingPolicy.FCFS)
|
||||
scheduler.requests = {waiting_request.request_id: waiting_request}
|
||||
scheduler.policy = SchedulingPolicy.FCFS
|
||||
scheduler.chunk_transfer_adapter = FakeAdapter()
|
||||
scheduler.input_coordinator = None
|
||||
scheduler.log_stats = False
|
||||
scheduler.scheduler_config = SimpleNamespace(enable_chunked_prefill=True)
|
||||
scheduler.num_lookahead_tokens = 0
|
||||
scheduler.kv_cache_manager = SimpleNamespace(
|
||||
new_step_starts=lambda: None,
|
||||
allocate_slots=lambda *args, **kwargs: SimpleNamespace(get_block_ids=lambda: ([1],)),
|
||||
get_num_common_prefix_blocks=lambda request_id: [0],
|
||||
take_new_block_ids=lambda: None,
|
||||
)
|
||||
scheduler.kv_cache_config = SimpleNamespace(kv_cache_groups=[object()])
|
||||
scheduler.use_v2_model_runner = use_v2_model_runner
|
||||
scheduler._retains_state_across_chunks = False
|
||||
scheduler.needs_kv_cache_zeroing = False
|
||||
scheduler.finished_req_ids = set()
|
||||
scheduler.encoder_cache_manager = SimpleNamespace(get_freed_mm_hashes=lambda: [])
|
||||
scheduler.connector = None
|
||||
scheduler.ec_connector = None
|
||||
scheduler.prev_step_scheduled_req_ids = set()
|
||||
scheduler._pending_finish_reqs = []
|
||||
scheduler._consume_pending_connector_output = lambda model_mode: None
|
||||
scheduler._process_pending_input_timeouts = lambda: None
|
||||
scheduler._make_cached_request_data = lambda **kwargs: SimpleNamespace(
|
||||
req_ids=[],
|
||||
resumed_req_ids=[],
|
||||
new_token_ids=[],
|
||||
all_token_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
num_output_tokens=[],
|
||||
)
|
||||
scheduler._update_after_schedule = lambda output: None
|
||||
scheduler._wrap_omni_scheduler_output = lambda output: output
|
||||
return scheduler
|
||||
|
||||
|
||||
def _chunk_request(request_id, **kwargs):
|
||||
defaults = dict(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=[1],
|
||||
num_computed_tokens=0,
|
||||
num_in_flight_tokens=0,
|
||||
status=None,
|
||||
sampling_params=None,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
lora_request=None,
|
||||
prompt_is_token_ids=True,
|
||||
additional_information=None,
|
||||
external_req_id=request_id,
|
||||
prefill_stats=None,
|
||||
record_event=lambda *args, **kwargs: None,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return SimpleNamespace(**defaults)
|
||||
|
||||
|
||||
def test_chunk_lifecycle_no_resubmit_and_state_survives_requeue(monkeypatch) -> None:
|
||||
# Native plane: a completed payload without a new chunk is never executed
|
||||
# again; requeued requests keep WAITING_FOR_CHUNK and their token counts.
|
||||
scheduler = _make_generation_scheduler(_chunk_request("unused"))
|
||||
monkeypatch.setattr("vllm_omni.core.sched.omni_generation_scheduler.create_request_queue", create_request_queue)
|
||||
scheduler._native_data_plane = True
|
||||
scheduler.chunk_transfer_adapter = None
|
||||
scheduler.input_coordinator = SimpleNamespace(finished_requests=set(), restore_queues=lambda *args, **kwargs: None)
|
||||
scheduler.running = []
|
||||
scheduler.waiting = create_request_queue(scheduler.policy)
|
||||
scheduler.skipped_waiting = create_request_queue(scheduler.policy)
|
||||
|
||||
completed = Request("completed", [1, 2], SamplingParams(max_tokens=4), pooling_params=None)
|
||||
completed.status = RequestStatus.RUNNING
|
||||
completed.num_computed_tokens = 2
|
||||
scheduler.running = [completed]
|
||||
scheduler.requests = {"completed": completed}
|
||||
scheduler.input_coordinator.finished_requests.add("completed") # terminal payload, no new chunk
|
||||
|
||||
assert scheduler._is_done_receiving_chunks("completed")
|
||||
assert not scheduler._is_done_receiving_chunks("unknown")
|
||||
output = scheduler.schedule()
|
||||
assert not output.num_scheduled_tokens
|
||||
assert scheduler._pending_finish_reqs == [completed]
|
||||
|
||||
completed.status = RequestStatus.WAITING_FOR_CHUNK
|
||||
scheduler._pending_finish_reqs = []
|
||||
scheduler._requeue_completed_native_chunks()
|
||||
assert completed.status == RequestStatus.WAITING_FOR_CHUNK
|
||||
assert completed.num_computed_tokens == 2
|
||||
assert not scheduler.running
|
||||
|
||||
|
||||
def test_generation_scheduler_schedules_terminal_empty_prompt_chunk_once(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
waiting = _chunk_request(
|
||||
"terminal",
|
||||
prompt_token_ids=[],
|
||||
num_prompt_tokens=0,
|
||||
additional_information=serialize_additional_information(
|
||||
{"codes": {"audio": torch.tensor([[1, 2, 3]], dtype=torch.long)}}
|
||||
),
|
||||
)
|
||||
scheduler = _make_generation_scheduler(waiting)
|
||||
scheduler.chunk_transfer_adapter.done_request_ids.add("terminal")
|
||||
scheduler.requests = {"terminal": waiting}
|
||||
assert _has_async_chunk_payload_to_run(waiting)
|
||||
# A terminal codec request with an empty prompt is still scheduled exactly once.
|
||||
monkeypatch.setattr("vllm_omni.core.sched.omni_generation_scheduler.create_request_queue", create_request_queue)
|
||||
|
||||
output = OmniGenerationScheduler.schedule(scheduler)
|
||||
|
||||
assert output.num_scheduled_tokens == {"terminal": 1}
|
||||
assert output.scheduled_new_reqs[0].req_id == "terminal"
|
||||
assert scheduler._pending_finish_reqs == []
|
||||
|
||||
|
||||
class TestRestoreQueuesOnError:
|
||||
"""Verify that restore_queues is called even when rewrapping raises."""
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Tests for ``OmniARScheduler._drop_aborted_queued_requests``.
|
||||
|
||||
``schedule()`` sweeps ``FINISHED_ABORTED`` requests out of the queues before
|
||||
@@ -68,6 +71,7 @@ def test_schedule_sweeps_skipped_waiting_before_upstream_selection(
|
||||
aborted = _StubRequest("req-aborted", RequestStatus.FINISHED_ABORTED)
|
||||
scheduler = _make_scheduler(policy, skipped_waiting=[aborted])
|
||||
scheduler.chunk_transfer_adapter = None
|
||||
scheduler.max_num_running_reqs = 8
|
||||
scheduler.input_coordinator = None
|
||||
scheduler.num_waiting_for_streaming_input = 0
|
||||
scheduler._process_pending_omni_inputs = lambda model_mode: None
|
||||
|
||||
@@ -7,6 +7,7 @@ window fills with stale entries after K completions.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -15,6 +16,7 @@ import pytest
|
||||
# Request / RequestStatus are bound in this module. Ruff isort would reorder them.
|
||||
# isort: off
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.v1.request import RequestStatus
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
|
||||
# isort: on
|
||||
@@ -72,3 +74,85 @@ def test_free_request_is_safe_without_a_chunk_transfer_adapter():
|
||||
sched._free_request(request) # must not raise
|
||||
|
||||
sched._free_input_coordinator_request.assert_called_once_with("req-1")
|
||||
|
||||
|
||||
def test_native_mooncake_finish_uses_confirmed_token_boundary() -> None:
|
||||
sched = _make_scheduler(chunk_transfer_adapter=None)
|
||||
sched.vllm_config = SimpleNamespace(kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector"))
|
||||
observed = {}
|
||||
|
||||
def connector_finished(request):
|
||||
observed["status"] = request.status
|
||||
observed["num_computed_tokens"] = request.num_computed_tokens
|
||||
return True, None
|
||||
|
||||
sched._connector_finished = connector_finished
|
||||
request = _make_finished_request("req-native")
|
||||
request.client_index = 0
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
request.kv_transfer_params = {
|
||||
"do_remote_decode": True,
|
||||
"transfer_id": "xfer-req-native",
|
||||
}
|
||||
request.num_computed_tokens = 13
|
||||
request.num_output_placeholders = 1
|
||||
request.num_prompt_tokens = 10
|
||||
|
||||
kv_params, _ = sched._free_request(request)
|
||||
|
||||
assert observed == {
|
||||
"status": RequestStatus.FINISHED_LENGTH_CAPPED,
|
||||
"num_computed_tokens": 12,
|
||||
}
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
assert request.num_computed_tokens == 13
|
||||
assert kv_params == {
|
||||
"transfer_id": "xfer-req-native",
|
||||
"num_transfer_tokens": 12,
|
||||
}
|
||||
|
||||
|
||||
def test_native_mooncake_recycles_ar_pages_in_physical_order() -> None:
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched.vllm_config = SimpleNamespace(kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector"))
|
||||
blocks = [SimpleNamespace(block_id=7), SimpleNamespace(block_id=2), SimpleNamespace(block_id=4)]
|
||||
sched.kv_cache_manager = SimpleNamespace(
|
||||
enable_caching=False,
|
||||
pop_blocks_for_free=MagicMock(return_value=blocks),
|
||||
block_pool=SimpleNamespace(free_blocks=MagicMock()),
|
||||
)
|
||||
sched.defer_block_free = False
|
||||
sched.processed_step_seq = 3
|
||||
request = SimpleNamespace(last_sched_seq=3)
|
||||
|
||||
sched._free_request_blocks(request)
|
||||
|
||||
sched.kv_cache_manager.block_pool.free_blocks.assert_called_once()
|
||||
freed = sched.kv_cache_manager.block_pool.free_blocks.call_args.args[0]
|
||||
assert [block.block_id for block in freed] == [2, 4, 7]
|
||||
|
||||
|
||||
def test_native_mooncake_deferred_ar_pages_drain_in_physical_order() -> None:
|
||||
from collections import deque
|
||||
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched.vllm_config = SimpleNamespace(kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector"))
|
||||
blocks = [SimpleNamespace(block_id=7), SimpleNamespace(block_id=2), SimpleNamespace(block_id=4)]
|
||||
sched.kv_cache_manager = SimpleNamespace(
|
||||
enable_caching=False,
|
||||
pop_blocks_for_free=MagicMock(return_value=blocks),
|
||||
block_pool=SimpleNamespace(free_blocks=MagicMock()),
|
||||
)
|
||||
sched.defer_block_free = True
|
||||
sched.processed_step_seq = 2
|
||||
sched.sched_step_seq = 3
|
||||
sched.deferred_frees = deque()
|
||||
request = SimpleNamespace(last_sched_seq=3)
|
||||
|
||||
sched._free_request_blocks(request)
|
||||
assert [block.block_id for block in sched.deferred_frees[0][1]] == [7, 4, 2]
|
||||
|
||||
sched.processed_step_seq = 3
|
||||
sched._drain_deferred_frees()
|
||||
freed = list(sched.kv_cache_manager.block_pool.free_blocks.call_args.args[0])
|
||||
assert [block.block_id for block in freed] == [2, 4, 7]
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Tests for the AR sampled-token logprob contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -93,6 +96,7 @@ class _Request:
|
||||
self.num_computed_tokens = 0
|
||||
self.num_in_flight_tokens = 0
|
||||
self.num_output_placeholders = 0
|
||||
self.output_token_ids: list[int] = []
|
||||
# vLLM 0.27 (a0c092ee72): Request gained num_stale_output_tokens to
|
||||
# track in-flight outputs discarded at preemption/streaming-stop.
|
||||
self.num_stale_output_tokens = 0
|
||||
@@ -145,6 +149,14 @@ def _make_scheduler_stub(requests: list[_Request]) -> SimpleNamespace:
|
||||
kv_cache_manager=SimpleNamespace(take_events=lambda: None),
|
||||
kv_event_publisher=SimpleNamespace(publish=lambda _events: None),
|
||||
recompute_kv_load_failures=False,
|
||||
_native_data_plane=False,
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
use_v2_model_runner=False,
|
||||
async_chunk=False,
|
||||
final_output=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
for name in _MIXIN_UPDATE_HELPERS:
|
||||
setattr(scheduler, name, MethodType(getattr(OmniSchedulerMixin, name), scheduler))
|
||||
@@ -167,10 +179,15 @@ def _bind_request_lifecycle(
|
||||
scheduler.requests.pop(request.request_id, None)
|
||||
scheduler.finished_req_ids.add(request.request_id)
|
||||
scheduler.finished_req_ids_dict[request.client_index].add(request.request_id)
|
||||
# vLLM 0.26 contract: (kv_xfer_params, ec_xfer_params)
|
||||
# Current vLLM contract: (kv_xfer_params, ec_xfer_params)
|
||||
return None, None
|
||||
|
||||
scheduler._update_request_with_output = update_request
|
||||
def update_with_history(request, token_ids):
|
||||
accepted, stopped = update_request(request, token_ids)
|
||||
request.output_token_ids.extend(accepted)
|
||||
return accepted, stopped
|
||||
|
||||
scheduler._update_request_with_output = update_with_history
|
||||
scheduler._process_kv_transfer_trigger = lambda _request, _tokens: False
|
||||
scheduler._handle_stopped_request = handle_stopped or (lambda _request: True)
|
||||
scheduler._free_request = free_request
|
||||
@@ -219,6 +236,7 @@ def test_mid_step_stop_trims_logprob_rows_with_token_ids() -> None:
|
||||
(output,) = outputs[0].outputs
|
||||
|
||||
assert output.new_token_ids == [7]
|
||||
assert output.num_generation_tokens == 1
|
||||
assert output.finish_reason is FinishReason.STOP
|
||||
token_rows = np.asarray(output.new_logprobs.logprob_token_ids)
|
||||
value_rows = np.asarray(output.new_logprobs.logprobs)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import _should_emit_engine_output
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("async_chunk", "final_output", "use_v2", "stopped", "has_control", "expected"),
|
||||
[
|
||||
(True, False, True, False, False, False), # async+v2 suppresses data
|
||||
(True, False, False, False, False, True), # v1 continues to emit
|
||||
(True, False, True, True, False, True), # stopped always emits
|
||||
(True, False, True, False, True, True), # control-only still emits
|
||||
],
|
||||
)
|
||||
def test_async_chunk_intermediate_stage_emits_only_control_outputs(
|
||||
async_chunk: bool,
|
||||
final_output: bool,
|
||||
use_v2: bool,
|
||||
stopped: bool,
|
||||
has_control: bool,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
model_config = SimpleNamespace(
|
||||
async_chunk=async_chunk,
|
||||
final_output=final_output,
|
||||
use_v2_model_runner=use_v2,
|
||||
)
|
||||
|
||||
assert (
|
||||
_should_emit_engine_output(
|
||||
model_config,
|
||||
stopped=stopped,
|
||||
has_control=has_control,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
@@ -17,6 +17,7 @@ import pytest
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
|
||||
@@ -1261,3 +1262,47 @@ def test_context_overflow_emits_an_error_output_with_the_reason() -> None:
|
||||
|
||||
sched._emit_streaming_context_overflow_outputs(outputs)
|
||||
assert len(outputs[2]) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", [True, False])
|
||||
def test_async_chunk_reserves_parked_slots_during_ar_admission(monkeypatch, native) -> None:
|
||||
sched = _make_scheduler(stage_id=1)
|
||||
parked = SimpleNamespace(request_id="parked")
|
||||
sched.requests = {"parked": parked}
|
||||
sched.waiting = []
|
||||
sched.running = []
|
||||
sched._native_data_plane = native
|
||||
sched.use_v2_model_runner = native
|
||||
sched.max_num_running_reqs = 8
|
||||
sched.input_coordinator = (
|
||||
SimpleNamespace(_waiting_for_chunk_running=[parked], restore_queues=lambda _w, _r: None) if native else None
|
||||
)
|
||||
sched.chunk_transfer_adapter = (
|
||||
None
|
||||
if native
|
||||
else SimpleNamespace(
|
||||
waiting_for_chunk_running_requests=[parked],
|
||||
_held_non_active=[],
|
||||
process_pending_chunks=lambda *_a, **_kw: None,
|
||||
collect_failed_send_request_ids=lambda: {},
|
||||
restore_queues=lambda *_a, **_kw: None,
|
||||
postprocess_scheduler_output=lambda *_a, **_kw: None,
|
||||
)
|
||||
)
|
||||
sched._consume_pending_connector_output = lambda model_mode: None
|
||||
sched._process_pending_input_timeouts = lambda: None
|
||||
sched._should_defer_waiting_admission = lambda: False
|
||||
sched.get_finished_requests_needing_kv_transfer = lambda: {}
|
||||
sched._wrap_omni_scheduler_output = lambda output, **_kwargs: output
|
||||
observed_limits: list[int] = []
|
||||
|
||||
def fake_schedule(self, _throttle_prefills=False):
|
||||
observed_limits.append(self.max_num_running_reqs)
|
||||
return SimpleNamespace(scheduled_new_reqs=[])
|
||||
|
||||
monkeypatch.setattr(VLLMScheduler, "schedule", fake_schedule)
|
||||
|
||||
sched.schedule()
|
||||
|
||||
assert observed_limits == [7 if native else 8]
|
||||
assert sched.max_num_running_reqs == 8
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
@@ -141,6 +144,7 @@ def test_voxcpm2_unified_decode_graph_does_not_defer_without_cuda_graph(monkeypa
|
||||
|
||||
def test_unified_decode_graph_deferral_restores_waiting_queue(monkeypatch) -> None:
|
||||
scheduler = _make_scheduler()
|
||||
scheduler.max_num_running_reqs = 8
|
||||
scheduler.running = [_MockRequest("decode")]
|
||||
original_waiting_req = _MockRequest("waiting", status=RequestStatus.WAITING)
|
||||
deferred_by_upstream = _MockRequest("deferred-by-upstream", status=RequestStatus.WAITING)
|
||||
|
||||
@@ -123,6 +123,9 @@ def _make_fast_path_sched(*, defer_block_free: bool) -> tuple[MagicMock, Request
|
||||
|
||||
sched = MagicMock()
|
||||
sched.max_num_scheduled_tokens = 8
|
||||
sched.max_num_running_reqs = 8
|
||||
sched._native_data_plane = False
|
||||
sched._retains_state_across_chunks = False
|
||||
sched._pause_state = PauseState.UNPAUSED
|
||||
sched.requests = {request.request_id: request}
|
||||
sched.running = [request]
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from collections import defaultdict
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from vllm.v1.engine import FinishReason
|
||||
@@ -87,7 +91,7 @@ def test_full_payload_coordinator_matches_legacy_gate(monkeypatch, stage_id, asy
|
||||
|
||||
|
||||
def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
|
||||
calls = []
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
scheduler = _Scheduler()
|
||||
scheduler.waiting = ["waiting"]
|
||||
scheduler.running = ["running"]
|
||||
@@ -115,7 +119,7 @@ def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
|
||||
collect_failed_send_request_ids=_collect_failed_sends,
|
||||
)
|
||||
scheduler.input_coordinator = SimpleNamespace(
|
||||
restore_queues=lambda waiting: calls.append(("restore-full", waiting))
|
||||
restore_queues=lambda waiting, running: calls.append(("restore-full", waiting, running))
|
||||
)
|
||||
|
||||
scheduler._process_pending_omni_inputs("ar")
|
||||
@@ -130,7 +134,7 @@ def test_schedule_lifecycle_helpers_process_and_restore_both_input_paths():
|
||||
("chunk-timeouts", omni_scheduler_mixin.DEFAULT_INPUT_WAIT_TIMEOUT_S),
|
||||
("failed-sends",),
|
||||
("restore-chunks", scheduler.waiting, scheduler.running, scheduler.requests),
|
||||
("restore-full", scheduler.waiting),
|
||||
("restore-full", scheduler.waiting, scheduler.running),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Drain ready events once, merging live requests and filtering cancellations."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin
|
||||
from vllm_omni.outputs import OmniConnectorOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_ready_inbox_coalesces_live_events_and_drops_cancelled(mocker):
|
||||
scheduler = OmniSchedulerMixin()
|
||||
scheduler.requests, scheduler.waiting, scheduler.running = {"r": object()}, [], []
|
||||
coordinator = SimpleNamespace(
|
||||
_async_chunk=True, update_request_metadata=mocker.Mock(), process_pending_chunks=mocker.Mock()
|
||||
)
|
||||
scheduler.input_coordinator = coordinator
|
||||
scheduler._init_omni_connector_output_inbox()
|
||||
scheduler.enqueue_omni_connector_output(OmniConnectorOutput(chunk_ready_req_ids={"r"}))
|
||||
scheduler.enqueue_omni_connector_output(
|
||||
OmniConnectorOutput(
|
||||
chunk_ready_req_ids={"r", "aborted"},
|
||||
chunk_finished_req_ids={"r", "aborted"},
|
||||
request_metadata={"r": {"decode_token_end": 2}, "aborted": {"decode_token_end": 99}},
|
||||
)
|
||||
)
|
||||
scheduler._consume_pending_connector_output(model_mode="ar")
|
||||
coordinator.update_request_metadata.assert_called_once_with(
|
||||
scheduler.requests, {"r": {"decode_token_end": 2}}, model_mode="ar"
|
||||
)
|
||||
coordinator.process_pending_chunks.assert_called_once_with([], [], {"r"}, {"r"})
|
||||
scheduler._consume_pending_connector_output(model_mode="ar")
|
||||
assert coordinator.update_request_metadata.call_count == 1
|
||||
coordinator.process_pending_chunks.assert_called_with([], [], set(), set())
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Tests for the ``num_waiting_for_streaming_input`` resync placed in
|
||||
``OmniARScheduler.finish_requests`` / ``OmniGenerationScheduler.finish_requests``
|
||||
and in ``OmniARScheduler.schedule``.
|
||||
@@ -128,6 +131,7 @@ def test_ar_schedule_resyncs_before_delegating_upstream(monkeypatch: pytest.Monk
|
||||
scheduler.skipped_waiting = []
|
||||
scheduler.num_waiting_for_streaming_input = 1 # leaked by the status rewrite
|
||||
scheduler.policy = "fcfs"
|
||||
scheduler.max_num_running_reqs = 8
|
||||
|
||||
seen: dict[str, int] = {}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for OmniSchedulingCoordinator.
|
||||
|
||||
These tests use mock request objects and mock queues. They do not require
|
||||
@@ -20,7 +20,9 @@ import torch
|
||||
import vllm_omni.core.sched.omni_scheduling_coordinator as coord_mod
|
||||
from vllm_omni.core.sched.omni_scheduling_coordinator import (
|
||||
OmniSchedulingCoordinator,
|
||||
uses_native_mrv2_data_plane,
|
||||
)
|
||||
from vllm_omni.core.sched.output import OmniChunkRecvHandle
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -56,6 +58,7 @@ def _make_request(req_id: str, status: str = "waiting") -> SimpleNamespace:
|
||||
prompt_token_ids=[],
|
||||
num_prompt_tokens=0,
|
||||
num_computed_tokens=0,
|
||||
num_output_placeholders=0,
|
||||
_all_token_ids=[],
|
||||
_output_token_ids=[],
|
||||
payload_sender_info=None,
|
||||
@@ -83,7 +86,7 @@ class MockQueue:
|
||||
def prepend_requests(self, requests):
|
||||
self._items = list(requests) + self._items
|
||||
|
||||
def remove(self, request):
|
||||
def remove_request(self, request):
|
||||
self._items.remove(request)
|
||||
|
||||
def remove_requests(self, requests):
|
||||
@@ -96,7 +99,47 @@ class MockQueue:
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
|
||||
class TestCoordinatorUpdateRequestMetadata(unittest.TestCase):
|
||||
class TestNativeMRV2DataPlaneSelection(unittest.TestCase):
|
||||
def test_native_plane_requires_v2_and_async_chunk_capability(self):
|
||||
native = SimpleNamespace(async_chunk=True, supports_native_mrv2_data_plane=True)
|
||||
|
||||
self.assertTrue(uses_native_mrv2_data_plane(native, use_v2_model_runner=True))
|
||||
self.assertFalse(uses_native_mrv2_data_plane(native, use_v2_model_runner=False))
|
||||
self.assertFalse(uses_native_mrv2_data_plane(SimpleNamespace(async_chunk=False), use_v2_model_runner=True))
|
||||
|
||||
|
||||
def test_chunk_registration_ready_and_terminal_lifecycle():
|
||||
coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
|
||||
req = _make_request("internal", status=RequestStatus.WAITING)
|
||||
req.external_req_id = "external"
|
||||
waiting = MockQueue([req])
|
||||
coord.process_pending_chunks(waiting, [], set(), set())
|
||||
assert req.status == RequestStatus.WAITING_FOR_CHUNK
|
||||
[handle] = coord.pending_chunk_registrations
|
||||
assert isinstance(handle, OmniChunkRecvHandle)
|
||||
assert (handle.request_id, handle.external_req_id) == ("internal", "external")
|
||||
coord.restore_queues(waiting, [])
|
||||
coord.process_pending_chunks(waiting, [], {"internal"}, set())
|
||||
assert req.status == RequestStatus.WAITING
|
||||
assert "internal" in coord.requests_with_ready_chunks
|
||||
req.status = RequestStatus.WAITING_FOR_CHUNK
|
||||
coord.process_pending_chunks(waiting, [], set(), {"internal"})
|
||||
assert "internal" in coord.finished_requests
|
||||
|
||||
|
||||
def test_chunk_waiting_removes_request_from_running_list():
|
||||
coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
|
||||
req = _make_request("running", status=RequestStatus.RUNNING)
|
||||
running = [req]
|
||||
|
||||
coord.process_pending_chunks(MockQueue(), running, set(), set())
|
||||
|
||||
assert running == []
|
||||
assert req.status == RequestStatus.WAITING_FOR_CHUNK
|
||||
assert list(coord._waiting_for_chunk_running) == [req]
|
||||
|
||||
|
||||
class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase):
|
||||
"""Test update_request_metadata applies scheduling metadata to requests."""
|
||||
|
||||
def test_ar_mode_no_longer_sets_additional_information(self):
|
||||
@@ -382,17 +425,18 @@ class TestTimeoutDetection(unittest.TestCase):
|
||||
self.assertEqual(result, {"r1"})
|
||||
self.assertEqual(len(coord._waiting_for_input), 0)
|
||||
|
||||
def test_free_finished_request_clears_waiting_since(self):
|
||||
"""free_finished_request clears coordinator lifecycle markers."""
|
||||
coord = OmniSchedulingCoordinator(stage_id=1)
|
||||
coord._waiting_since["r1"] = 0.0
|
||||
coord._full_payload_input_received.add("r1")
|
||||
def test_free_finished_request_clears_all_lifecycle_state(self):
|
||||
"""free_finished_request makes stale connector events harmless."""
|
||||
coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
|
||||
r1, r2 = _make_request("r1"), _make_request("r2")
|
||||
coord.finished_requests.add("r1")
|
||||
coord.requests_with_ready_chunks.add("r1")
|
||||
coord._waiting_for_chunk_running.extend([r1, r2])
|
||||
coord.pending_chunk_registrations = [OmniChunkRecvHandle(request_id="r1"), OmniChunkRecvHandle(request_id="r2")]
|
||||
|
||||
coord.free_finished_request("r1")
|
||||
self.assertNotIn("r1", coord._waiting_since)
|
||||
self.assertNotIn("r1", coord._full_payload_input_received)
|
||||
|
||||
self.assertNotIn("r1", coord.finished_requests)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
self.assertNotIn("r1", coord.requests_with_ready_chunks)
|
||||
self.assertEqual([r.request_id for r in coord._waiting_for_chunk_running], ["r2"])
|
||||
self.assertEqual([h.request_id for h in coord.pending_chunk_registrations], ["r2"])
|
||||
+1778
-894
File diff suppressed because it is too large.
Load diff
@@ -1,119 +0,0 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.core.prefix_cache import OmniTensorPrefixCache
|
||||
|
||||
# `cpu` is the lane selector, not a hardware claim: the only Buildkite steps that
|
||||
# reach tests/ select `core_model and cpu`, and they run on GPU hardware (l4_1).
|
||||
# Marking this `cuda` collects it nowhere. Real hardware need is the skipif below.
|
||||
pytestmark = [
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cpu,
|
||||
pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a CUDA device"),
|
||||
]
|
||||
|
||||
NUM_BLOCKS = 10
|
||||
BLOCK_SIZE = 4
|
||||
HIDDEN_SIZE = 2
|
||||
DTYPE = torch.float32
|
||||
|
||||
|
||||
def get_omni_pcache() -> OmniTensorPrefixCache:
|
||||
"""Build an OmniTensorPrefixCache, but don't init mm tensors."""
|
||||
return OmniTensorPrefixCache(
|
||||
num_blocks=NUM_BLOCKS,
|
||||
block_size=BLOCK_SIZE,
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
hs_dtype=DTYPE,
|
||||
)
|
||||
|
||||
|
||||
def test_drain_ready_async_writes_is_a_noop_before_the_event_is_ready(monkeypatch):
|
||||
"""drain_ready_async_writes() must not touch the cache while the
|
||||
scheduled D2H's event has not completed, and must consume it exactly
|
||||
once after it has.
|
||||
|
||||
Real GPU timing (e.g. via torch.cuda._sleep to stall the copy stream)
|
||||
was tried and found unreliable on this box: cross-stream wait_stream/
|
||||
wait_event did not reliably delay event.query() from reporting ready
|
||||
immediately, likely a virtualization/driver quirk of this cloud GPU.
|
||||
Forcing the event's query() result directly tests the branching logic
|
||||
in drain_ready_async_writes/_consume_pending_write deterministically,
|
||||
independent of real CUDA scheduling behavior.
|
||||
"""
|
||||
cache = get_omni_pcache()
|
||||
mm_key = "codes.audio"
|
||||
cache.maybe_init_missing_mm_cache_keys({mm_key: torch.zeros(4, HIDDEN_SIZE)}, seq_len=4)
|
||||
|
||||
num_tokens = 4
|
||||
slot_mapping_gpu = torch.arange(8, 8 + num_tokens, dtype=torch.int64, device="cuda")
|
||||
mm_gpu = {
|
||||
mm_key: torch.arange(num_tokens * HIDDEN_SIZE, dtype=torch.float32, device="cuda").reshape(
|
||||
num_tokens, HIDDEN_SIZE
|
||||
)
|
||||
}
|
||||
|
||||
cache.schedule_async_write(
|
||||
hidden_states_gpu=None,
|
||||
multimodal_outputs_gpu=mm_gpu,
|
||||
slot_mapping_gpu=slot_mapping_gpu,
|
||||
num_tokens_unpadded=num_tokens,
|
||||
num_tokens_padded=num_tokens,
|
||||
)
|
||||
|
||||
pending_event = cache._pending_write.event
|
||||
rows = cache.mm_outputs_cache[mm_key].view(-1, HIDDEN_SIZE)
|
||||
|
||||
monkeypatch.setattr(pending_event, "query", lambda: False)
|
||||
assert cache.drain_ready_async_writes() == 0
|
||||
assert torch.all(rows[8 : 8 + num_tokens] == 0)
|
||||
assert cache._pending_write is not None
|
||||
|
||||
monkeypatch.setattr(pending_event, "query", lambda: True)
|
||||
assert cache.drain_ready_async_writes() == 1
|
||||
assert torch.equal(rows[8 : 8 + num_tokens], mm_gpu[mm_key].cpu())
|
||||
assert cache._pending_write is None
|
||||
|
||||
# Nothing pending anymore: a second drain is a no-op, not a re-scatter.
|
||||
assert cache.drain_ready_async_writes() == 0
|
||||
|
||||
|
||||
def test_schedule_async_write_consumes_previous_pending_write_first():
|
||||
"""Scheduling step N+1's write must first consume (scatter) step N's
|
||||
pending write, even if the caller never explicitly called
|
||||
drain_ready_async_writes() in between -- otherwise step N's data would
|
||||
be silently dropped when overwritten by step N+1's pending write."""
|
||||
cache = get_omni_pcache()
|
||||
mm_key = "codes.audio"
|
||||
cache.maybe_init_missing_mm_cache_keys({mm_key: torch.zeros(4, HIDDEN_SIZE)}, seq_len=4)
|
||||
|
||||
num_tokens = 4
|
||||
first_slots = torch.arange(8, 8 + num_tokens, dtype=torch.int64, device="cuda")
|
||||
first_gpu = {mm_key: torch.full((num_tokens, HIDDEN_SIZE), 1.0, device="cuda")}
|
||||
second_slots = torch.arange(12, 12 + num_tokens, dtype=torch.int64, device="cuda")
|
||||
second_gpu = {mm_key: torch.full((num_tokens, HIDDEN_SIZE), 2.0, device="cuda")}
|
||||
|
||||
cache.schedule_async_write(
|
||||
hidden_states_gpu=None,
|
||||
multimodal_outputs_gpu=first_gpu,
|
||||
slot_mapping_gpu=first_slots,
|
||||
num_tokens_unpadded=num_tokens,
|
||||
num_tokens_padded=num_tokens,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# This call's internal _consume_pending_write() must flush the FIRST
|
||||
# write before the second one is scheduled; the caller never drained.
|
||||
cache.schedule_async_write(
|
||||
hidden_states_gpu=None,
|
||||
multimodal_outputs_gpu=second_gpu,
|
||||
slot_mapping_gpu=second_slots,
|
||||
num_tokens_unpadded=num_tokens,
|
||||
num_tokens_padded=num_tokens,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
cache.drain_ready_async_writes()
|
||||
|
||||
rows = cache.mm_outputs_cache[mm_key].view(-1, HIDDEN_SIZE)
|
||||
assert torch.all(rows[8:12] == 1.0)
|
||||
assert torch.all(rows[12:16] == 2.0)
|
||||
@@ -0,0 +1,182 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU tests for PrefixCacheRunnerMixin: one-shot construction, policy
|
||||
snapshot identity/isolation, and the shared save/materialize gates.
|
||||
No vLLM and no GPU required (same shim as test_prefix_cache.py)."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
try: # pragma: no cover - shim only matters on vllm-less dev machines
|
||||
import vllm # noqa: F401
|
||||
except ModuleNotFoundError:
|
||||
_root = Path(__file__).resolve().parents[2]
|
||||
for _pkg in ("vllm_omni", "vllm_omni.core"):
|
||||
if _pkg not in sys.modules:
|
||||
_m = ModuleType(_pkg)
|
||||
_m.__path__ = [str(_root / _pkg.replace(".", "/"))]
|
||||
sys.modules[_pkg] = _m
|
||||
import logging
|
||||
|
||||
_vllm = ModuleType("vllm")
|
||||
_vllm_logger = ModuleType("vllm.logger")
|
||||
setattr(_vllm_logger, "init_logger", logging.getLogger)
|
||||
setattr(_vllm, "logger", _vllm_logger)
|
||||
sys.modules["vllm"] = _vllm
|
||||
sys.modules["vllm.logger"] = _vllm_logger
|
||||
|
||||
from vllm_omni.core.prefix_cache.interface import PrefixCacheConfig, StageCacheOutputs
|
||||
from vllm_omni.core.prefix_cache.runner_mixin import PrefixCacheRunnerMixin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
BLOCK_SIZE = 4
|
||||
|
||||
|
||||
def _patch_pp(monkeypatch, is_last: bool) -> None:
|
||||
"""Route the mixin's call-time get_pp_group import to a fake rank."""
|
||||
mod = ModuleType("vllm.distributed.parallel_state")
|
||||
setattr(mod, "get_pp_group", lambda: SimpleNamespace(is_last_rank=is_last))
|
||||
if "vllm.distributed" not in sys.modules:
|
||||
monkeypatch.setitem(sys.modules, "vllm.distributed", ModuleType("vllm.distributed"))
|
||||
monkeypatch.setitem(sys.modules, "vllm.distributed.parallel_state", mod)
|
||||
|
||||
|
||||
def _full_attention_groups():
|
||||
"""A single-group config that passes check_prefix_cache_kv_groups.
|
||||
|
||||
Real FullAttentionSpec when vllm is installed; otherwise a stand-in
|
||||
registered under the import path the check resolves at call time.
|
||||
"""
|
||||
try:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
except ModuleNotFoundError:
|
||||
iface = sys.modules.get("vllm.v1.kv_cache_interface")
|
||||
if iface is None:
|
||||
iface = ModuleType("vllm.v1.kv_cache_interface")
|
||||
|
||||
class _FullAttentionSpecStub:
|
||||
pass
|
||||
|
||||
setattr(iface, "FullAttentionSpec", _FullAttentionSpecStub)
|
||||
sys.modules["vllm.v1"] = ModuleType("vllm.v1")
|
||||
sys.modules["vllm.v1.kv_cache_interface"] = iface
|
||||
FullAttentionSpec = getattr(iface, "FullAttentionSpec")
|
||||
return [SimpleNamespace(kv_cache_spec=FullAttentionSpec.__new__(FullAttentionSpec))]
|
||||
|
||||
|
||||
class _FakeBlockTables:
|
||||
"""Group-0 table whose layout attrs pass check_prefix_cache_block_layout."""
|
||||
|
||||
def __init__(self):
|
||||
self.block_tables = [object()]
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return SimpleNamespace()
|
||||
|
||||
|
||||
class _Runner(PrefixCacheRunnerMixin):
|
||||
def __init__(self, cfg: PrefixCacheConfig | None = None):
|
||||
# Any: tests swap in _CacheStub for the manager.
|
||||
self.omni_prefix_cache: Any = None
|
||||
self._omni_prefix_cache_cfg = cfg
|
||||
self.input_batch = SimpleNamespace(block_table=_FakeBlockTables(), req_ids=[])
|
||||
self.kv_cache_config = SimpleNamespace(kv_cache_groups=_full_attention_groups())
|
||||
self.is_pooling_model = False
|
||||
|
||||
|
||||
def _cfg() -> PrefixCacheConfig:
|
||||
return PrefixCacheConfig(num_blocks=8, block_size=BLOCK_SIZE)
|
||||
|
||||
|
||||
def _sched():
|
||||
return SimpleNamespace(scheduled_new_reqs=(), finished_req_ids=set(), num_scheduled_tokens={})
|
||||
|
||||
|
||||
class _CacheStub:
|
||||
def __init__(self):
|
||||
self.save_calls = []
|
||||
self.materialize_calls = []
|
||||
self.outs = StageCacheOutputs(hidden_states=None, mm_outputs={})
|
||||
|
||||
def save_outputs(self, hidden, mm, *, num_tokens_unpadded, num_tokens_padded):
|
||||
self.save_calls.append((hidden, mm, num_tokens_unpadded, num_tokens_padded))
|
||||
return 7
|
||||
|
||||
def materialize(self, step_id, req_ids):
|
||||
self.materialize_calls.append((step_id, req_ids))
|
||||
return self.outs
|
||||
|
||||
|
||||
def test_step_begin_builds_once_and_registers_snapshot_policy(monkeypatch):
|
||||
_patch_pp(monkeypatch, is_last=True)
|
||||
r = _Runner(_cfg())
|
||||
r._snapshot_prefix_cache_model_policy(
|
||||
SimpleNamespace(requires_full_prefix_cached_hidden_states=True, deferred_prefix_cache_mm_keys=("k",))
|
||||
)
|
||||
r._prefix_cache_step_begin(_sched())
|
||||
mgr = r.omni_prefix_cache
|
||||
assert mgr is not None and r._omni_prefix_cache_cfg is None
|
||||
assert mgr._policy is r._omni_cache_policy # the load-time snapshot, same object
|
||||
r._prefix_cache_step_begin(_sched())
|
||||
assert r.omni_prefix_cache is mgr # one-shot
|
||||
|
||||
|
||||
def test_default_policy_isolated_per_instance():
|
||||
a, b = _Runner(), _Runner()
|
||||
a._snapshot_prefix_cache_model_policy(SimpleNamespace(requires_full_prefix_cached_hidden_states=False))
|
||||
assert a._omni_cache_policy.needs_full_hidden_states is False
|
||||
assert a._model_needs_full_prefix_hidden_states() is False
|
||||
assert b._omni_cache_policy.needs_full_hidden_states is True # class default untouched
|
||||
assert PrefixCacheRunnerMixin._omni_cache_policy.needs_full_hidden_states is True
|
||||
|
||||
|
||||
def test_non_last_rank_consumes_cfg_without_building(monkeypatch):
|
||||
_patch_pp(monkeypatch, is_last=False)
|
||||
r = _Runner(_cfg())
|
||||
r._prefix_cache_step_begin(_sched())
|
||||
assert r.omni_prefix_cache is None and r._omni_prefix_cache_cfg is None
|
||||
|
||||
|
||||
def test_disabled_cache_step_begin_is_a_noop():
|
||||
r = _Runner(cfg=None) # enable_prefix_caching off: nothing staged
|
||||
r._prefix_cache_step_begin(_sched())
|
||||
assert r.omni_prefix_cache is None
|
||||
|
||||
|
||||
def test_save_step_gates_and_passthrough(monkeypatch):
|
||||
r = _Runner()
|
||||
stub = _CacheStub()
|
||||
hidden = torch.zeros(2, 2)
|
||||
|
||||
def save():
|
||||
return r._prefix_cache_save_step(hidden, None, num_tokens_unpadded=2, num_tokens_padded=2)
|
||||
|
||||
_patch_pp(monkeypatch, is_last=True)
|
||||
assert save() is None # cache off
|
||||
r.omni_prefix_cache = stub
|
||||
r.is_pooling_model = True
|
||||
assert save() is None # pooling stage never writes
|
||||
r.is_pooling_model = False
|
||||
_patch_pp(monkeypatch, is_last=False)
|
||||
assert save() is None # not the last PP rank
|
||||
_patch_pp(monkeypatch, is_last=True)
|
||||
assert save() == 7
|
||||
assert stub.save_calls == [(hidden, {}, 2, 2)] # empty mm stays {}
|
||||
|
||||
|
||||
def test_materialize_requires_explicit_step_and_snapshot_req_ids():
|
||||
r = _Runner()
|
||||
stub = _CacheStub()
|
||||
r.omni_prefix_cache = stub
|
||||
assert r._prefix_cache_materialize(None, ["a"]) == (None, None)
|
||||
assert stub.materialize_calls == [] # step_id None: nothing to consume
|
||||
hidden = {"a": torch.zeros(1, 2)}
|
||||
stub.outs = StageCacheOutputs(hidden_states=hidden, mm_outputs={})
|
||||
assert r._prefix_cache_materialize(7, ["a"]) == (hidden, None) # empty mm -> None
|
||||
assert stub.materialize_calls == [(7, ["a"])]
|
||||
@@ -54,7 +54,8 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": ["H100", "B200"],
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
|
||||
@@ -26,8 +26,8 @@
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
@@ -42,7 +42,73 @@
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.9731,
|
||||
1.1986,
|
||||
1.686
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
152.5665,
|
||||
270.2559,
|
||||
301.455
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1027.2953,
|
||||
3306.3735,
|
||||
4686.5366
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
458.1553,
|
||||
2030.2335,
|
||||
2068.6233
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2357,
|
||||
0.791,
|
||||
1.0324
|
||||
]
|
||||
},
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.5383,
|
||||
0.6042,
|
||||
0.7547
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
333.2633,
|
||||
533.8656,
|
||||
547.3388
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1857.2154,
|
||||
6600.5101,
|
||||
10450.6811
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
986.4666,
|
||||
3411.0754,
|
||||
3352.7515
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.4423,
|
||||
1.5734,
|
||||
2.3024
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -72,8 +138,8 @@
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
@@ -88,6 +154,16 @@
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -26,8 +26,8 @@
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -40,7 +40,41 @@
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.9731,
|
||||
1.1986
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
152.5665,
|
||||
270.2559
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1027.2953,
|
||||
3306.3735
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
458.1553,
|
||||
2030.2335
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2357,
|
||||
0.791
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -570,6 +570,7 @@ def test_omni_tpot_baseline_accepts_measured_finite_sample():
|
||||
@pytest.mark.parametrize(
|
||||
("num_tpot_samples", "mean_tpot_ms", "match"),
|
||||
[
|
||||
(None, 10.0, "no measurable TPOT samples"),
|
||||
(0, float("nan"), "no measurable TPOT samples"),
|
||||
(1, float("nan"), "mean_tpot_ms is not finite"),
|
||||
],
|
||||
@@ -577,14 +578,17 @@ def test_omni_tpot_baseline_accepts_measured_finite_sample():
|
||||
def test_omni_tpot_baseline_rejects_missing_or_nonfinite_sample(num_tpot_samples, mean_tpot_ms, match):
|
||||
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||
|
||||
result = {
|
||||
"completed": 1,
|
||||
"Hardware": "H100",
|
||||
"mean_tpot_ms": mean_tpot_ms,
|
||||
}
|
||||
if num_tpot_samples is not None:
|
||||
result["num_tpot_samples"] = num_tpot_samples
|
||||
|
||||
with pytest.raises(AssertionError, match=match):
|
||||
assert_result(
|
||||
{
|
||||
"completed": 1,
|
||||
"Hardware": "H100",
|
||||
"num_tpot_samples": num_tpot_samples,
|
||||
"mean_tpot_ms": mean_tpot_ms,
|
||||
},
|
||||
result,
|
||||
{"baseline": {"H100": {"mean_tpot_ms": 20.0}}},
|
||||
1,
|
||||
)
|
||||
@@ -323,12 +323,18 @@ def test_lingbot_like_sink_survives_sliding_window_eviction():
|
||||
kv = runner.kv_cache
|
||||
assert kv is not None
|
||||
|
||||
for _ in range(8):
|
||||
state = commit_one_frame(runner, "s1", "main")
|
||||
sink_block = kv.block_table(state.adapter("main"))[0]
|
||||
for _ in range(7):
|
||||
state = commit_one_frame(runner, "s1", "main")
|
||||
|
||||
table = kv.block_table(state.adapter("main"))
|
||||
assert table[0] != kv.null_block_id
|
||||
assert table[1] == kv.null_block_id
|
||||
adapter = state.adapter("main")
|
||||
table = kv.block_table(adapter)
|
||||
assert len(table) == 6 # one sink + five recent frames
|
||||
assert table[0] == sink_block
|
||||
assert kv.null_block_id not in table
|
||||
assert adapter.absolute_num_computed_tokens == 8 * BLOCK
|
||||
assert adapter.num_computed_tokens == 6 * BLOCK
|
||||
|
||||
ctx = state.get_kv_caches("main", seq_len=BLOCK, commit_current=False)[0].forward_ctx
|
||||
visible, _ = ctx.video_block_table(torch.device("cpu"))
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for the AR-Diffusion KV cache helpers (Phase 1, PR-2).
|
||||
|
||||
Covers the request adapter, the chunk-window spec/manager (registration + the
|
||||
@@ -269,9 +270,14 @@ def test_compiled_paged_write_does_not_clone_the_pool():
|
||||
# drives the reinplace pass, so a drift here would silently test nothing.
|
||||
real_schema = torch.ops.vllm_omni.ar_diffusion_paged_write_attn.default._schema
|
||||
real_mutated = [arg.name for arg in real_schema.arguments if arg.alias_info and arg.alias_info.is_write]
|
||||
assert real_mutated == ["key_pool", "value_pool"], (
|
||||
# The staging buffers are mutated too when reuse_history_staging is on, but they are separate
|
||||
# allocations and not what this test is about; the pools are what must not be cloned.
|
||||
assert real_mutated[:2] == ["key_pool", "value_pool"], (
|
||||
f"the real paged-write op now mutates {real_mutated}; this test still models two pools"
|
||||
)
|
||||
assert set(real_mutated) - {"key_pool", "value_pool"} <= {"stage_key", "stage_value"}, (
|
||||
f"unexpected mutated argument in the real paged-write op: {real_mutated}"
|
||||
)
|
||||
|
||||
num_blocks, heads, dim = 8, 4, 16
|
||||
cache_shape = (num_blocks, BLOCK, heads, dim)
|
||||
@@ -357,6 +363,7 @@ def _make_tiny_capacity_kv(
|
||||
available_bytes: int,
|
||||
gpu_memory_fraction: float = 1.0,
|
||||
model_owned_state_bytes_per_session: int = 0,
|
||||
reuse_history_staging: bool = False,
|
||||
) -> ARDiffusionKVCache:
|
||||
return ARDiffusionKVCache(
|
||||
ARDiffusionKVConfig(
|
||||
@@ -365,6 +372,7 @@ def _make_tiny_capacity_kv(
|
||||
window_chunks=3,
|
||||
sink_chunks=3,
|
||||
gpu_memory_fraction=gpu_memory_fraction,
|
||||
reuse_history_staging=reuse_history_staging,
|
||||
),
|
||||
num_layers=1,
|
||||
num_kv_heads=1,
|
||||
@@ -382,6 +390,33 @@ def _make_tiny_capacity_kv(
|
||||
)
|
||||
|
||||
|
||||
def test_history_staging_is_reserved_in_the_kv_budget(monkeypatch):
|
||||
"""The per-layer staging pair is worker-wide memory allocated after admission; the budget must hold it."""
|
||||
monkeypatch.setenv("VLLM_OMNI_AR_DIFFUSION_KV_GATHER", "1")
|
||||
# Same geometry as the capacity-two test: 192 bytes fits two sessions exactly without staging.
|
||||
plain = _make_tiny_capacity_kv(requested_capacity=2, available_bytes=192)
|
||||
assert plain.session_capacity == 2 and plain.history_staging_reserved_bytes == 0
|
||||
|
||||
staged = _make_tiny_capacity_kv(requested_capacity=2, available_bytes=192, reuse_history_staging=True)
|
||||
# 2 (K, V) x 1 layer x (sink 3 + window 3 + one action-capacity block) tokens x 1 head x 1 dim x 4 bytes.
|
||||
assert staged.history_staging_reserved_bytes == 2 * 1 * 7 * 1 * 1 * 4
|
||||
# It comes out of the same budget, so the second session no longer fits.
|
||||
assert staged.session_capacity == 1
|
||||
assert staged.num_blocks_total * 8 + staged.history_staging_reserved_bytes <= 192
|
||||
|
||||
# Boundary: one session fits at 128 bytes without staging; with staging the same budget is rejected
|
||||
# up front instead of failing on the first forward.
|
||||
_make_tiny_capacity_kv(requested_capacity=1, available_bytes=128)
|
||||
with pytest.raises(ValueError, match="cannot fit one session"):
|
||||
_make_tiny_capacity_kv(requested_capacity=1, available_bytes=128, reuse_history_staging=True)
|
||||
|
||||
|
||||
def test_history_staging_is_not_reserved_when_the_gather_path_is_off(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_OMNI_AR_DIFFUSION_KV_GATHER", raising=False)
|
||||
staged = _make_tiny_capacity_kv(requested_capacity=2, available_bytes=192, reuse_history_staging=True)
|
||||
assert staged.history_staging_reserved_bytes == 0 and staged.session_capacity == 2
|
||||
|
||||
|
||||
def test_capacity_two_retains_both_windows_and_allocates_next_block():
|
||||
# LingBot-like single-branch geometry: page=8 bytes; capacity=2 requires
|
||||
# 17 managed + 3 scratch blocks, plus two 16-byte cross-attention
|
||||
@@ -588,3 +623,70 @@ def test_non_contiguous_branch_indices_rejected():
|
||||
kv_branches=(ARDiffusionKVBranchSpec("main", 1),),
|
||||
session_capacity=1,
|
||||
)
|
||||
|
||||
|
||||
def test_staged_window_reuse_matches_a_full_gather():
|
||||
"""Reusing the staged history must be byte-identical to re-gathering the whole window.
|
||||
|
||||
This is the property the optimisation rests on: the leading blocks were staged by an earlier forward of
|
||||
the same AR block, so skipping them can only be correct if what is already there equals what a fresh
|
||||
gather would write. The test stages once, mutates only the current chunk's blocks in the pool (what a
|
||||
later probe of the same block does), restages the tail alone, and compares against a full gather.
|
||||
"""
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import _stage_window
|
||||
|
||||
torch.manual_seed(0)
|
||||
num_blocks, block_size, heads, dim = 6, 4, 2, 8
|
||||
current_blocks = 2
|
||||
key_cache = torch.randn(num_blocks, block_size, heads, dim)
|
||||
value_cache = torch.randn(num_blocks, block_size, heads, dim)
|
||||
block_ids = torch.tensor([5, 4, 3, 2, 1, 0])
|
||||
stage_key = torch.zeros(num_blocks * block_size, heads, dim)
|
||||
stage_value = torch.zeros(num_blocks * block_size, heads, dim)
|
||||
|
||||
_stage_window(stage_key, stage_value, key_cache, value_cache, block_ids, num_blocks, block_size, first_block=0)
|
||||
|
||||
# A later probe rewrites only the current chunk's blocks in the pool.
|
||||
for slot in block_ids[num_blocks - current_blocks :]:
|
||||
key_cache[slot] = torch.randn(block_size, heads, dim)
|
||||
value_cache[slot] = torch.randn(block_size, heads, dim)
|
||||
|
||||
_stage_window(
|
||||
stage_key,
|
||||
stage_value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_ids,
|
||||
num_blocks,
|
||||
block_size,
|
||||
first_block=num_blocks - current_blocks,
|
||||
)
|
||||
|
||||
full_key = torch.zeros_like(stage_key)
|
||||
full_value = torch.zeros_like(stage_value)
|
||||
_stage_window(full_key, full_value, key_cache, value_cache, block_ids, num_blocks, block_size, first_block=0)
|
||||
|
||||
torch.testing.assert_close(stage_key, full_key, rtol=0, atol=0)
|
||||
torch.testing.assert_close(stage_value, full_value, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_staged_window_tail_refresh_leaves_history_untouched():
|
||||
"""The tail restage must not write the history rows -- that is what makes it cheaper."""
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import _stage_window
|
||||
|
||||
torch.manual_seed(1)
|
||||
num_blocks, block_size, heads, dim = 5, 4, 1, 4
|
||||
key_cache = torch.randn(num_blocks, block_size, heads, dim)
|
||||
value_cache = torch.randn(num_blocks, block_size, heads, dim)
|
||||
block_ids = torch.arange(num_blocks)
|
||||
stage_key = torch.full((num_blocks * block_size, heads, dim), -1.0)
|
||||
stage_value = torch.full((num_blocks * block_size, heads, dim), -1.0)
|
||||
|
||||
_stage_window(
|
||||
stage_key, stage_value, key_cache, value_cache, block_ids, num_blocks, block_size, first_block=num_blocks - 1
|
||||
)
|
||||
|
||||
history_rows = (num_blocks - 1) * block_size
|
||||
assert (stage_key[:history_rows] == -1.0).all(), "history rows were rewritten by a tail restage"
|
||||
assert (stage_value[:history_rows] == -1.0).all()
|
||||
torch.testing.assert_close(stage_key[history_rows:], key_cache[-1].reshape(block_size, heads, dim), rtol=0, atol=0)
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
|
||||
from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVCache, ARDiffusionKVConfig
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
DIMS = dict(
|
||||
@@ -108,3 +109,104 @@ def test_state_close_frees_both_branch_blocks():
|
||||
state.close()
|
||||
# All blocks returned — no leak across an evicted session.
|
||||
assert kv.manager.block_pool.get_num_free_blocks() == free_total
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"frames_per_call,block_size,max_model_len,num_calls",
|
||||
[(1, 16, 113, 1000), (3, 16, 113, 1000), (3, 1560, 1 << 20, 241)],
|
||||
ids=["single-frame", "three-frame", "lingbot-two-minutes"],
|
||||
)
|
||||
@pytest.mark.parametrize("sink_chunks,reset_at_boundary", [(0, False), (2, False), (0, True), (2, True)])
|
||||
def test_rollout_keeps_metadata_bounded(
|
||||
frames_per_call, block_size, max_model_len, num_calls, sink_chunks, reset_at_boundary
|
||||
):
|
||||
"""Long rollouts retain exact global-position K/V with bounded storage indices."""
|
||||
kv = ARDiffusionKVCache(
|
||||
ARDiffusionKVConfig(
|
||||
enable=True,
|
||||
chunk_size=block_size,
|
||||
window_chunks=3,
|
||||
sink_chunks=sink_chunks,
|
||||
reset_at_boundary=reset_at_boundary,
|
||||
),
|
||||
num_layers=1,
|
||||
num_kv_heads=1,
|
||||
head_size=4,
|
||||
dtype=torch.float32,
|
||||
block_size=block_size,
|
||||
max_model_len=max_model_len,
|
||||
available_bytes=1 << 24,
|
||||
kv_branches=(ARDiffusionKVBranchSpec("main", 0),),
|
||||
session_capacity=1,
|
||||
frames_per_block=frames_per_call,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
adapter = kv.begin_request("long-rollout")
|
||||
state = ARDiffusionKVState(kv, "long-rollout", {"main": adapter}, num_layers=1)
|
||||
free_total = kv.manager.block_pool.get_num_free_blocks()
|
||||
pool_shape = kv.key_cache(0).shape
|
||||
retained_frames = sink_chunks + (0 if reset_at_boundary else 3)
|
||||
for call_index in range(num_calls):
|
||||
start = call_index * frames_per_call * block_size
|
||||
stop = start + frames_per_call * block_size
|
||||
if frames_per_call == 1:
|
||||
kv.allocate_chunk(adapter)
|
||||
slots = kv.chunk_write_slots(adapter)
|
||||
else:
|
||||
context = state.get_kv_caches("main", seq_len=stop - start, commit_current=True)[0].forward_ctx
|
||||
context.ensure_video_slots(torch.device("cpu"))
|
||||
slots = context.current_video_slot_mapping
|
||||
assert len(kv.block_table(adapter)) <= retained_frames + frames_per_call
|
||||
values = torch.arange(start, stop, dtype=torch.float32).view(-1, 1, 1).expand(-1, 1, 4)
|
||||
kv.key_cache(0).flatten(0, 1)[slots] = values
|
||||
kv.value_cache(0).flatten(0, 1)[slots] = -values
|
||||
if frames_per_call == 1:
|
||||
kv.commit_chunk(adapter)
|
||||
else:
|
||||
state.commit_paged_context("main")
|
||||
assert state._committed["main"] == stop
|
||||
sink_end = min(stop, sink_chunks * block_size)
|
||||
tail_start = stop if reset_at_boundary else max(sink_end, stop - 3 * block_size)
|
||||
expected = torch.cat((torch.arange(sink_end), torch.arange(tail_start, stop))).float()
|
||||
expected = expected.view(-1, 1, 1).expand(-1, 1, 4)
|
||||
resident = kv.window_block_ids(adapter)
|
||||
torch.testing.assert_close(kv.key_cache(0)[resident].flatten(0, 1), expected)
|
||||
torch.testing.assert_close(kv.value_cache(0)[resident].flatten(0, 1), -expected)
|
||||
assert adapter.completed_chunks * block_size == stop
|
||||
assert adapter.absolute_num_computed_tokens == stop
|
||||
assert len(kv.block_table(adapter)) <= retained_frames
|
||||
assert adapter.num_computed_tokens <= retained_frames * block_size
|
||||
assert kv.key_cache(0).shape == pool_shape
|
||||
assert free_total - kv.manager.block_pool.get_num_free_blocks() == min(stop // block_size, retained_frames)
|
||||
state.reset()
|
||||
fresh = state.adapter("main")
|
||||
assert fresh.compacted_tokens == fresh.absolute_num_computed_tokens == fresh.num_computed_tokens == 0
|
||||
assert kv.block_table(fresh) == []
|
||||
assert kv.manager.block_pool.get_num_free_blocks() == free_total
|
||||
kv.allocate_chunk(fresh)
|
||||
assert len(kv.chunk_write_slots(fresh)) == block_size
|
||||
state.close()
|
||||
assert kv.manager.block_pool.get_num_free_blocks() == free_total
|
||||
|
||||
|
||||
def test_compaction_offsets_are_request_local():
|
||||
kv = make_cache()
|
||||
free_total = kv.manager.block_pool.get_num_free_blocks()
|
||||
first, second = kv.begin_request("first"), kv.begin_request("second")
|
||||
for index in range(8):
|
||||
kv.allocate_chunk(first)
|
||||
kv.commit_chunk(first)
|
||||
if index in (0, 4):
|
||||
kv.allocate_chunk(second)
|
||||
kv.commit_chunk(second)
|
||||
assert first.absolute_num_computed_tokens == 8 * 16
|
||||
assert first.compacted_tokens == 6 * 16
|
||||
assert second.absolute_num_computed_tokens == 2 * 16
|
||||
assert second.compacted_tokens == 0
|
||||
assert first.num_computed_tokens == second.num_computed_tokens == 2 * 16
|
||||
assert set(kv.block_table(first)).isdisjoint(kv.block_table(second))
|
||||
kv.end_request(first)
|
||||
kv.end_request(second)
|
||||
group = kv.manager.coordinator.single_type_managers[0]
|
||||
assert not group.req_to_blocks and not group.num_cached_block
|
||||
assert kv.manager.block_pool.get_num_free_blocks() == free_total
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Tests for AR-Diffusion paged self-attention contexts."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -20,6 +21,8 @@ from vllm_omni.experimental.ar_diffusion.kv_cache import (
|
||||
ar_diffusion_paged_attention,
|
||||
paged_write_attn,
|
||||
)
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache import paged_attention as paged_attention_module
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.config import KV_GATHER_ENV
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
@@ -32,8 +35,12 @@ POS = "positive"
|
||||
NEG = "negative"
|
||||
|
||||
|
||||
def make_state(*, num_layers=1, window_chunks=2, dtype=torch.float32, device=torch.device("cpu")):
|
||||
cfg = ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks)
|
||||
def make_state(
|
||||
*, num_layers=1, window_chunks=2, dtype=torch.float32, device=torch.device("cpu"), reuse_history_staging=False
|
||||
):
|
||||
cfg = ARDiffusionKVConfig(
|
||||
enable=True, chunk_size=BLOCK, window_chunks=window_chunks, reuse_history_staging=reuse_history_staging
|
||||
)
|
||||
kv = ARDiffusionKVCache(
|
||||
cfg,
|
||||
num_layers=num_layers,
|
||||
@@ -236,6 +243,86 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len,
|
||||
assert st.adapter(POS).completed_chunks == before + (1 if commit_current else 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("history_chunks", [0, 1, 2, 3])
|
||||
def test_staged_reuse_refreshes_the_current_blocks_at_their_live_offset(monkeypatch, history_chunks):
|
||||
"""A second probe of the same AR block restages its current K/V where the table actually holds it.
|
||||
|
||||
The padded table always ends in at least one action-capacity block, and while the window is still
|
||||
growing in unused window capacity too, so "the last current_blocks entries" is padding. The refresh
|
||||
has to start right after the visible history: empty (window growing), partial, full, and after a slide.
|
||||
"""
|
||||
monkeypatch.setenv(KV_GATHER_ENV, "1")
|
||||
device = torch.device("cpu")
|
||||
dtype = torch.float32
|
||||
kv, st = make_state(dtype=dtype, device=device, window_chunks=2, reuse_history_staging=True)
|
||||
if history_chunks:
|
||||
_commit_video_span(kv, st, kv_branch=POS, n_chunks=history_chunks, dtype=dtype, device=device)
|
||||
|
||||
ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
key_cache, value_cache = kv.key_cache(0), kv.value_cache(0)
|
||||
|
||||
def probe(step: int):
|
||||
# Each probe of the block writes new current K/V, then stages.
|
||||
kv._k_pools[0][ctx.current_video_slot_mapping] = torch.full((BLOCK, N_HEADS, HEAD_DIM), float(step))
|
||||
kv._v_pools[0][ctx.current_video_slot_mapping] = torch.full((BLOCK, N_HEADS, HEAD_DIM), -float(step))
|
||||
# Store the metadata on the context the way prepare() does.
|
||||
(ctx.block_table, ctx.query_start_loc, ctx.seq_lens, ctx.max_query_len, ctx.max_seq_len) = (
|
||||
ctx.build_block_table(action_len=0, query_len=BLOCK, device=device)
|
||||
)
|
||||
block_table, max_seq_len = ctx.block_table, ctx.max_seq_len
|
||||
ctx._prepare_history_staging(0)
|
||||
n_blocks = max_seq_len // BLOCK
|
||||
block_ids = block_table[0, :n_blocks].to(torch.long)
|
||||
stage_k, stage_v = ctx.history_staging(0)
|
||||
paged_attention_module._stage_window(
|
||||
stage_k,
|
||||
stage_v,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_ids,
|
||||
n_blocks,
|
||||
BLOCK,
|
||||
first_block=ctx.stage_first_block if ctx.reuse_history else 0,
|
||||
)
|
||||
full_k = key_cache.index_select(0, block_ids).reshape(n_blocks * BLOCK, N_HEADS, HEAD_DIM)
|
||||
full_v = value_cache.index_select(0, block_ids).reshape(n_blocks * BLOCK, N_HEADS, HEAD_DIM)
|
||||
return stage_k, stage_v, full_k, full_v, n_blocks
|
||||
|
||||
stage_k, stage_v, full_k, full_v, n_blocks = probe(1)
|
||||
assert ctx.reuse_history is False
|
||||
assert torch.equal(stage_k, full_k) and torch.equal(stage_v, full_v)
|
||||
|
||||
stage_k, stage_v, full_k, full_v, n_blocks = probe(2)
|
||||
assert ctx.reuse_history is True
|
||||
visible_history_blocks = min(history_chunks, 2 - 1) # window of 2 blocks minus the current one
|
||||
assert ctx.stage_first_block == visible_history_blocks
|
||||
# The padded table is wider than the live window, so the end-of-table guess is not the offset.
|
||||
assert n_blocks - 1 != ctx.stage_first_block
|
||||
assert torch.equal(stage_k, full_k) and torch.equal(stage_v, full_v)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gather_enabled", [False, True])
|
||||
def test_staging_is_only_allocated_for_the_gather_path(monkeypatch, gather_enabled):
|
||||
if gather_enabled:
|
||||
monkeypatch.setenv(KV_GATHER_ENV, "1")
|
||||
else:
|
||||
monkeypatch.delenv(KV_GATHER_ENV, raising=False)
|
||||
device = torch.device("cpu")
|
||||
kv, st = make_state(device=device, reuse_history_staging=True)
|
||||
# The manager owns the pairs and allocates them only when their consumer is on.
|
||||
assert bool(kv.history_staging) is gather_enabled
|
||||
ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
(ctx.block_table, ctx.query_start_loc, ctx.seq_lens, ctx.max_query_len, ctx.max_seq_len) = ctx.build_block_table(
|
||||
action_len=0, query_len=BLOCK, device=device
|
||||
)
|
||||
ctx._prepare_history_staging(0)
|
||||
assert ctx.staging_enabled is gather_enabled
|
||||
stage_k, _ = ctx.history_staging(0)
|
||||
assert (stage_k is not None) is gather_enabled
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _gpu_flash_attn_usable(), reason="usable GPU FlashAttention is required")
|
||||
@pytest.mark.parametrize("history_chunks", [1, 3])
|
||||
@pytest.mark.parametrize("action_len", [0, 3])
|
||||
@@ -376,13 +463,27 @@ def test_custom_op_registration_idempotent():
|
||||
assert hasattr(torch.ops.vllm_omni, "ar_diffusion_paged_write_attn")
|
||||
|
||||
|
||||
def test_custom_op_compiles_fullgraph_without_recompile_on_value_change():
|
||||
def test_custom_op_mutable_arguments_cannot_be_elided_as_defaults():
|
||||
# Older PyTorch ADInplaceOrView handlers index positional mutable inputs
|
||||
# directly. Default-valued trailing inputs can disappear before that handler.
|
||||
schema = torch.ops.vllm_omni.ar_diffusion_paged_write_attn.default._schema
|
||||
mutable = [arg for arg in schema.arguments if arg.alias_info is not None and arg.alias_info.is_write]
|
||||
assert {arg.name for arg in mutable} == {"key_pool", "value_pool", "stage_key", "stage_value"}
|
||||
assert all(not arg.has_default_value() for arg in mutable)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reuse_history_staging", [False, True])
|
||||
def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(monkeypatch, reuse_history_staging):
|
||||
"""The op must trace as one opaque node: fullgraph OK, and changed tensor
|
||||
VALUES (new slots / block ids) must not trigger recompilation."""
|
||||
import torch._dynamo
|
||||
|
||||
device = torch.device("cpu")
|
||||
kv, st = make_state(num_layers=2, window_chunks=2)
|
||||
monkeypatch.setenv(KV_GATHER_ENV, "1")
|
||||
kv, st = make_state(num_layers=2, window_chunks=2, reuse_history_staging=reuse_history_staging)
|
||||
if reuse_history_staging:
|
||||
# Hold the host-side staging offset fixed while table/slot values change.
|
||||
_commit_video_span(kv, st, kv_branch=POS, n_chunks=2, dtype=torch.float32, device=device)
|
||||
|
||||
def run_one_forward(commit):
|
||||
contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=commit)
|
||||
@@ -391,9 +492,8 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change():
|
||||
q = torch.randn(BLOCK, N_HEADS, HEAD_DIM)
|
||||
k = torch.randn(BLOCK, N_HEADS, HEAD_DIM)
|
||||
v = torch.randn(BLOCK, N_HEADS, HEAD_DIM)
|
||||
# Both layers through ONE compiled fn: layer_idx is a tensor, so a
|
||||
# different layer must NOT recompile (all 40 DiT blocks share the
|
||||
# block-forward code object in production).
|
||||
# Both layers use one compiled function; the tensor-valued layer index
|
||||
# must not specialize it.
|
||||
for layer_ctx in contexts:
|
||||
out = compiled(layer_ctx.to_layer_inputs(), q, k, v)
|
||||
st.commit_paged_context(POS)
|
||||
@@ -419,3 +519,52 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change():
|
||||
# Leave a clean dynamo state for later suites in the same pytest process
|
||||
# (e.g. model_executor transformers models).
|
||||
torch._dynamo.reset()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _gpu_flash_attn_usable(), reason="usable GPU FlashAttention is required")
|
||||
@pytest.mark.parametrize("history_chunks", [0, 1, 3])
|
||||
@pytest.mark.parametrize("action_len", [0, 3])
|
||||
def test_contiguous_kv_gather_path_matches_paged_path_gpu(monkeypatch, history_chunks, action_len):
|
||||
"""VLLM_OMNI_AR_DIFFUSION_KV_GATHER=1 gathers the visible blocks and runs varlen FA3 without a block table.
|
||||
|
||||
Covers an empty, partial and full window plus a partially filled action block:
|
||||
the gather reads the tail-padding null block, so its (zeroed) rows must be masked.
|
||||
"""
|
||||
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.float16
|
||||
kv, st = make_state(dtype=dtype, device=device, window_chunks=2)
|
||||
if history_chunks:
|
||||
_commit_video_span(kv, st, kv_branch=POS, n_chunks=history_chunks, dtype=dtype, device=device)
|
||||
|
||||
layer_ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0]
|
||||
ctx = layer_ctx.forward_ctx
|
||||
current_k = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
current_v = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
action_k = action_v = None
|
||||
if action_len:
|
||||
action_k = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
action_v = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
query = torch.randn(1, BLOCK + action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
ctx.prepare(device=device, action_len=action_len, query_len=query.shape[1])
|
||||
inputs = layer_ctx.to_layer_inputs()
|
||||
|
||||
def run() -> torch.Tensor:
|
||||
return paged_write_attn(
|
||||
inputs,
|
||||
query[0],
|
||||
current_k[0],
|
||||
current_v[0],
|
||||
action_k[0] if action_len else None,
|
||||
action_v[0] if action_len else None,
|
||||
HEAD_DIM**-0.5,
|
||||
).unsqueeze(0)
|
||||
|
||||
monkeypatch.delenv(KV_GATHER_ENV, raising=False)
|
||||
paged = run()
|
||||
monkeypatch.setenv(KV_GATHER_ENV, "1")
|
||||
gathered = run()
|
||||
assert torch.isfinite(gathered).all()
|
||||
# Same kernel family on the same K/V: only accumulation order differs.
|
||||
torch.testing.assert_close(gathered, paged, rtol=2e-3, atol=2e-3)
|
||||
@@ -16,13 +16,14 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl
|
||||
from vllm_omni.diffusion.attention.backends.utils import fa # noqa: E402
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
|
||||
|
||||
is_gpu = current_omni_platform.is_cuda_alike() or current_omni_platform.is_xpu()
|
||||
HAS_FLASH_ATTN = fa.HAS_FLASH_ATTN
|
||||
@@ -42,10 +43,12 @@ flash_attn_func = fa.flash_attn_func # noqa: N813
|
||||
(9, "4", frozenset((2, 3, 4)), 4),
|
||||
],
|
||||
)
|
||||
@pytest.mark.cpu
|
||||
def test_versioned_flash_attention_selection(device_major, requested, supported, expected):
|
||||
assert fa._choose_vllm_flash_attn_version(device_major, requested, supported) == expected
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_versioned_flash_attention_selection_rejects_invalid_or_unavailable_version():
|
||||
with pytest.raises(ValueError, match="must be 2, 3, or 4"):
|
||||
fa._choose_vllm_flash_attn_version(9, "5", frozenset((2, 3, 4)))
|
||||
@@ -92,7 +95,8 @@ def pad_tensor(tensor: torch.Tensor, target_seq_len: int, pad_value: float = 0.0
|
||||
return torch.cat([tensor, padding], dim=1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
|
||||
def test_padding_equivalence():
|
||||
"""
|
||||
Case 1: Test that padded and unpadded inputs produce similar outputs.
|
||||
@@ -185,7 +189,8 @@ def test_padding_equivalence():
|
||||
print("✓ Case 1 PASSED: Padded and unpadded outputs are very close!")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
|
||||
def test_fa_vs_sdpa():
|
||||
"""
|
||||
Case 2: Compare FlashAttention and SDPA backends with padding.
|
||||
@@ -302,7 +307,8 @@ def test_fa_vs_sdpa():
|
||||
print("✓ Case 2 PASSED: FA and SDPA outputs are very close!")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
|
||||
def test_flash_attn_func_preferred_over_varlen():
|
||||
"""Test flash_attn_func availability and basic forward call."""
|
||||
if not HAS_FLASH_ATTN:
|
||||
@@ -332,7 +338,8 @@ def test_flash_attn_func_preferred_over_varlen():
|
||||
print("✓ flash_attn_func forward works correctly!")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
|
||||
@pytest.mark.parametrize("k_len", [40, 256])
|
||||
def test_cross_attn_key_padding_vs_sdpa(k_len):
|
||||
"""
|
||||
@@ -390,6 +397,7 @@ def test_cross_attn_key_padding_vs_sdpa(k_len):
|
||||
print("✓ Case 3 PASSED: FA and SDPA cross-attention outputs are very close!")
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_varlen_masked_routing_by_role(monkeypatch):
|
||||
"""The unpad route is picked by role, not Q/K length equality; runs without a GPU."""
|
||||
calls = []
|
||||
@@ -424,6 +432,7 @@ def test_varlen_masked_routing_by_role(monkeypatch):
|
||||
assert cu_seqlens_k.tolist() == [0, 2, 5]
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch):
|
||||
calls = []
|
||||
|
||||
@@ -454,6 +463,7 @@ def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch):
|
||||
assert calls[0]["softmax_scale"] == 0.5
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_packed_varlen_metadata_bypasses_mask_unpadding(monkeypatch):
|
||||
calls = []
|
||||
|
||||
@@ -486,6 +496,7 @@ def test_packed_varlen_metadata_bypasses_mask_unpadding(monkeypatch):
|
||||
assert calls[0][3]["max_seqlen_q"] == 6
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_packed_varlen_metadata_must_be_complete(monkeypatch):
|
||||
monkeypatch.setattr(fa, "HAS_FLASH_ATTN", True)
|
||||
impl = FlashAttentionImpl(num_heads=2, head_size=4, softmax_scale=0.5, causal=False)
|
||||
@@ -542,6 +553,7 @@ def _fake_torch_npu(monkeypatch, *, npu_fusion_attention=None):
|
||||
# --- Test group A: boundary resolution (_resolve_packed_seq_npu) -------------
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_resolve_packed_seq_accepts_real_plus_pad_contract():
|
||||
impl = _npu_impl()
|
||||
q = torch.randn(1, 8, 2, 4)
|
||||
@@ -551,6 +563,7 @@ def test_resolve_packed_seq_accepts_real_plus_pad_contract():
|
||||
assert impl._resolve_packed_seq_npu(q, q, extra) == ([5, 8], [5, 8])
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_resolve_packed_seq_single_document_no_padding():
|
||||
impl = _npu_impl()
|
||||
q = torch.randn(1, 8, 2, 4)
|
||||
@@ -574,6 +587,7 @@ def test_resolve_packed_seq_single_document_no_padding():
|
||||
"pad_longer_than_real",
|
||||
],
|
||||
)
|
||||
@pytest.mark.cpu
|
||||
def test_resolve_packed_seq_rejects_malformed_metadata(case):
|
||||
"""Any metadata outside the exact [real, pad] contract returns None so the
|
||||
caller falls back to the masked path."""
|
||||
@@ -630,6 +644,7 @@ def _npu_impl_with_mocked_paths() -> tuple[FlashAttentionImpl, torch.Tensor]:
|
||||
return impl, sentinel
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_env_unset_uses_varlen(monkeypatch):
|
||||
monkeypatch.delenv("MINDIE_SD_FA_TYPE", raising=False)
|
||||
_fake_mindiesd(monkeypatch)
|
||||
@@ -642,6 +657,7 @@ def test_npu_env_unset_uses_varlen(monkeypatch):
|
||||
impl._forward_prefix_kv_slice_npu.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_env_laser_uses_slice(monkeypatch):
|
||||
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
|
||||
_fake_mindiesd(monkeypatch)
|
||||
@@ -655,6 +671,7 @@ def test_npu_env_laser_uses_slice(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fa_type", ["prompt_flash_attn", "fused_attn_score", "ascend_laser_attenton", "garbage"])
|
||||
@pytest.mark.cpu
|
||||
def test_npu_env_non_laser_value_falls_back_to_varlen(monkeypatch, fa_type):
|
||||
"""Locks in the CURRENT dispatch behavior (PR #5891).
|
||||
|
||||
@@ -675,6 +692,7 @@ def test_npu_env_non_laser_value_falls_back_to_varlen(monkeypatch, fa_type):
|
||||
impl._forward_prefix_kv_slice_npu.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_varlen_opt_in_unset_takes_mask_path(monkeypatch):
|
||||
"""Models that do not set ``npu_attn_varlen`` (e.g. Wan/Cosmos) keep using
|
||||
the existing masked attention_forward path; the new branches are skipped."""
|
||||
@@ -702,6 +720,7 @@ def test_npu_varlen_opt_in_unset_takes_mask_path(monkeypatch):
|
||||
("query_length", "key_length", "expected_inner_precise"),
|
||||
[(4, 4, 0), (2, 4, 0), (4, 2, 2)],
|
||||
)
|
||||
@pytest.mark.cpu
|
||||
def test_npu_causal_uses_native_right_down_mode(
|
||||
monkeypatch,
|
||||
query_length,
|
||||
@@ -746,6 +765,7 @@ def test_npu_causal_uses_native_right_down_mode(
|
||||
assert out is expected_out
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_causal_composes_explicit_keep_mask(monkeypatch):
|
||||
expected_out = torch.randn(1, 4, 2, 4)
|
||||
fusion_attention = Mock(return_value=(expected_out, None, None, None, 0, 0, 0))
|
||||
@@ -782,6 +802,7 @@ def test_npu_causal_composes_explicit_keep_mask(monkeypatch):
|
||||
assert out is expected_out
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_noncausal_without_explicit_mask_stays_unmasked(monkeypatch):
|
||||
captured: dict = {}
|
||||
|
||||
@@ -800,6 +821,7 @@ def test_npu_noncausal_without_explicit_mask_stays_unmasked(monkeypatch):
|
||||
# --- Test group D: laser input pre-scaling in _forward_prefix_kv_slice_npu --
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_prefix_kv_slice_applies_laser_input_scaling(monkeypatch):
|
||||
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
|
||||
captured: dict = {}
|
||||
@@ -838,6 +860,7 @@ def test_prefix_kv_slice_applies_laser_input_scaling(monkeypatch):
|
||||
torch.testing.assert_close(out, torch.ones(1, 8, 2, 4) * 256.0)
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_prefix_kv_slice_no_scaling_without_factor(monkeypatch):
|
||||
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
|
||||
captured: dict = {}
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Cross-repository CPU dispatch contracts, requiring the matching MindIE-SD source.
|
||||
|
||||
Only native operators are replaced. Public adapters, rotations, layouts, padding,
|
||||
scale preparation, sequence metadata and output cropping execute their real code.
|
||||
Operator substitutes carry floating tensors; these are not numerical quantization tests.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
|
||||
|
||||
mindiesd = pytest.importorskip("mindiesd")
|
||||
torch_npu = importlib.import_module("torch_npu")
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"precision,batch,q_len,kv_len,q_heads",
|
||||
[
|
||||
("fp8", 1, 130, 130, 2),
|
||||
("mxfp8", 1, 130, 130, 2),
|
||||
("mxfp4", 1, 130, 130, 2),
|
||||
("mxfp8", 2, 17, 8, 4),
|
||||
("mxfp8", 2, 8, 17, 4),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["BSND", "BNSD"])
|
||||
def test_omni_calls_real_dense_public_api(monkeypatch, precision, layout, batch, q_len, kv_len, q_heads):
|
||||
assert callable(getattr(mindiesd, "quant_attention", None)), "MindIE-SD must export quant_attention"
|
||||
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
|
||||
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
|
||||
quantized_inputs = []
|
||||
|
||||
def dynamic_quant(tensor, **kwargs):
|
||||
quantized_inputs.append((tensor.clone(), kwargs))
|
||||
shape = list(tensor.shape)
|
||||
axis = kwargs.get("axis", -1)
|
||||
shape[axis] = max(1, shape[axis] // 32)
|
||||
return tensor, torch.ones(shape, dtype=torch.uint8)
|
||||
|
||||
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", dynamic_quant, raising=False)
|
||||
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", dynamic_quant, raising=False)
|
||||
execute = Mock(side_effect=lambda query, *args, **kwargs: (query, None))
|
||||
metadata = Mock(return_value=torch.zeros(1, dtype=torch.int32))
|
||||
if precision == "fp8":
|
||||
monkeypatch.setattr(torch.ops.mindiesd, "fused_infer_attention_score_v2", execute, raising=False)
|
||||
elif precision == "mxfp8":
|
||||
monkeypatch.setattr(torch_npu, "npu_fused_infer_attention_score_v2", execute, raising=False)
|
||||
else:
|
||||
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn", execute, raising=False)
|
||||
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn_metadata", metadata, raising=False)
|
||||
|
||||
query = torch.randn(batch, q_len, q_heads, 64, dtype=torch.bfloat16)
|
||||
key = torch.randn(batch, kv_len, 2, 64, dtype=torch.bfloat16)
|
||||
value = torch.randn_like(key)
|
||||
if layout == "BNSD":
|
||||
query, key, value = (tensor.transpose(1, 2) for tensor in (query, key, value))
|
||||
impl = flash_attn.FlashAttentionImpl(
|
||||
num_heads=q_heads,
|
||||
head_size=64,
|
||||
softmax_scale=0.37,
|
||||
qkv_layout=layout,
|
||||
)
|
||||
output = impl.forward_npu(query, key, value, AttentionMetadata(extra={"kv_cache_dtype": precision}))
|
||||
rotation = get_quant_attention_rotation(query.device, query.dtype, 64)
|
||||
expected = query @ rotation
|
||||
torch.testing.assert_close(output, expected)
|
||||
assert output.shape == query.shape and len(quantized_inputs) == 3
|
||||
execute.assert_called_once()
|
||||
kwargs = execute.call_args.kwargs
|
||||
assert kwargs["softmax_scale"] == 0.37
|
||||
if precision == "fp8":
|
||||
assert kwargs["input_layout"] == "BNSD"
|
||||
assert quantized_inputs[0][0].shape == (2, 130, 64)
|
||||
assert quantized_inputs[0][1]["row_block_size"] == 128
|
||||
elif precision == "mxfp8":
|
||||
args = execute.call_args.args
|
||||
assert args[0].shape == (batch * q_len, q_heads, 64)
|
||||
assert args[1].shape == args[2].shape == (batch * kv_len, 2, 64)
|
||||
assert kwargs["input_layout"] == "TND"
|
||||
assert kwargs["num_query_heads"] == q_heads and kwargs["num_key_value_heads"] == 2
|
||||
assert kwargs["actual_seq_qlen"] == [q_len * (i + 1) for i in range(batch)]
|
||||
assert kwargs["actual_seq_kvlen"] == [kv_len * (i + 1) for i in range(batch)]
|
||||
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, 0]
|
||||
else:
|
||||
seq_axis = 1 if layout == "BSND" else 2
|
||||
assert quantized_inputs[0][0].shape[seq_axis] == 512
|
||||
assert kwargs["seqused_q"].tolist() == kwargs["seqused_kv"].tolist() == [512]
|
||||
assert kwargs["layout_q"] == kwargs["layout_out"] == layout
|
||||
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
|
||||
assert kwargs["q_descale_dtype"] == torch_npu.float8_e8m0fnu
|
||||
assert execute.call_args.args[3].shape[:2] == (1, 2)
|
||||
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, seq_axis]
|
||||
assert kwargs["metadata"] is metadata.return_value
|
||||
metadata.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("precision", ["fp8", "mxfp4"])
|
||||
@pytest.mark.parametrize("length", [65, 600])
|
||||
def test_omni_calls_real_sparse_public_api(monkeypatch, precision, length):
|
||||
rf = importlib.import_module("mindiesd.layers.flash_attn.sparse_flash_attn_rf_v3")
|
||||
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
|
||||
for module in (flash_attn, rainfusion_attn):
|
||||
monkeypatch.setattr(module, "get_current_diffusion_config_or_none", lambda: None)
|
||||
rainfusion_attn._mindiesd_supports_precision.cache_clear()
|
||||
monkeypatch.setattr(rf, "do_tensor_rearrange_only", lambda q, k, v, *a, **kw: (q, k, v))
|
||||
monkeypatch.setattr(rf, "avgpool", lambda *a, **kw: torch.ones(1))
|
||||
monkeypatch.setattr(rf, "_generate_mask_direct", lambda *a, **kw: torch.ones(1, 2, 8, 8))
|
||||
monkeypatch.setattr(rf, "_bsa_inv_rearrange", lambda out, *a: out)
|
||||
|
||||
def quantize(tensor, **kwargs):
|
||||
shape = list(tensor.shape)
|
||||
axis = kwargs.get("axis", -1)
|
||||
shape[axis] = max(1, shape[axis] // 32)
|
||||
return tensor, torch.ones(shape, dtype=torch.uint8)
|
||||
|
||||
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", quantize, raising=False)
|
||||
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", quantize, raising=False)
|
||||
execute = Mock(side_effect=lambda **kw: (torch.ones_like(kw["query"]), None))
|
||||
monkeypatch.setattr(torch.ops.mindiesd, "block_sparse_attention", execute, raising=False)
|
||||
query = torch.randn(1, length, 2, 64, dtype=torch.bfloat16)
|
||||
impl = rainfusion_attn.RainFusionAttentionImpl(
|
||||
num_heads=2,
|
||||
head_size=64,
|
||||
softmax_scale=0.37,
|
||||
qkv_layout="BSND",
|
||||
backend_kwargs={"sparsity": 0.8},
|
||||
)
|
||||
plan = rainfusion_attn.RainFusionPlan(used_len=length, prefix_len=0, latent_shape=(1, 1, length))
|
||||
try:
|
||||
output = impl._forward_sparse_npu(
|
||||
query,
|
||||
query,
|
||||
query,
|
||||
plan,
|
||||
AttentionMetadata(extra={"kv_cache_dtype": precision}),
|
||||
)
|
||||
finally:
|
||||
rainfusion_attn._mindiesd_supports_precision.cache_clear()
|
||||
torch.testing.assert_close(output, torch.ones_like(query))
|
||||
execute.assert_called_once()
|
||||
kwargs = execute.call_args.kwargs
|
||||
assert kwargs["actual_seq_lengths"] == kwargs["actual_seq_lengths_kv"] == [length]
|
||||
assert kwargs["q_input_layout"] == "BNSD"
|
||||
assert kwargs["block_sparse_mask"] is not None
|
||||
if precision == "mxfp4":
|
||||
assert kwargs["query"].shape[2] == (length + 63) // 64 * 64
|
||||
assert kwargs["quant_mode"] == 2
|
||||
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
|
||||
assert kwargs["v_scale_dtype"] == torch_npu.float8_e8m0fnu
|
||||
@@ -0,0 +1,323 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU contract tests for MindIE adapters; no NPU kernels are mocked in production."""
|
||||
|
||||
import sys
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention import layer as layer_mod
|
||||
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
_DEFAULT_SKIP_STEPS = frozenset({0, 1, 38, 39})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runtime(monkeypatch):
|
||||
mod = ModuleType("mindiesd")
|
||||
|
||||
def execute(q, k, v, *, precision, layout, scale, q_rot=None, k_rot=None):
|
||||
return q
|
||||
|
||||
setattr(mod, "quant_attention", Mock(side_effect=execute))
|
||||
monkeypatch.setitem(sys.modules, mod.__name__, mod)
|
||||
for module in (flash_attn, layer_mod):
|
||||
monkeypatch.setattr(module, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
|
||||
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
|
||||
monkeypatch.setattr(rainfusion_attn, "get_current_diffusion_config_or_none", lambda: None)
|
||||
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: False)
|
||||
supports_precision = rainfusion_attn._mindiesd_supports_precision
|
||||
supports_precision.cache_clear()
|
||||
yield mod
|
||||
supports_precision.cache_clear()
|
||||
|
||||
|
||||
def flash(layout="BSND"):
|
||||
return flash_attn.FlashAttentionImpl(
|
||||
num_heads=2,
|
||||
head_size=64,
|
||||
softmax_scale=0.37,
|
||||
qkv_layout=layout,
|
||||
)
|
||||
|
||||
|
||||
def sparse(**kwargs):
|
||||
return rainfusion_attn.RainFusionAttentionImpl(
|
||||
num_heads=2,
|
||||
head_size=64,
|
||||
softmax_scale=0.37,
|
||||
qkv_layout="BSND",
|
||||
backend_kwargs={"sparsity": 0.8, **kwargs},
|
||||
)
|
||||
|
||||
|
||||
def video_metadata(**extra):
|
||||
return AttentionMetadata(
|
||||
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32), used_len=4096),
|
||||
extra={"max_seqlen_q": 4096, **extra},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
|
||||
@pytest.mark.parametrize("layout", [None, "BSND", "BNSD"])
|
||||
def test_exact_runtime_layout_scale_and_rotation(runtime, method, layout):
|
||||
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
|
||||
if layout == "BNSD":
|
||||
q = q.transpose(1, 2)
|
||||
impl = flash(layout=layout)
|
||||
metadata = AttentionMetadata(extra={"kv_cache_dtype": method})
|
||||
assert impl.forward_npu(q, q, q, metadata) is q
|
||||
args, kwargs = runtime.quant_attention.call_args
|
||||
assert args[0] is q # no unconditional transpose or extra quantization
|
||||
assert kwargs["layout"] == (layout or "BSND") and kwargs["scale"] == 0.37 and kwargs["precision"] == method
|
||||
assert "attn_mask" not in kwargs
|
||||
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
|
||||
|
||||
assert kwargs["q_rot"] is kwargs["k_rot"]
|
||||
torch.testing.assert_close(kwargs["q_rot"], get_quant_attention_rotation(q.device, q.dtype, 64))
|
||||
|
||||
|
||||
def test_missing_runtime_requires_config_change(runtime):
|
||||
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
|
||||
del runtime.quant_attention
|
||||
with pytest.raises(ImportError, match="diffusion_kv_cache_dtype"):
|
||||
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
|
||||
|
||||
|
||||
def test_operator_failure_never_falls_back(runtime):
|
||||
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
|
||||
runtime.quant_attention.side_effect = RuntimeError("operator failure")
|
||||
with pytest.raises(RuntimeError, match="operator failure"):
|
||||
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
|
||||
runtime.quant_attention.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
|
||||
def test_unsupported_shape_requires_config_change(runtime, method):
|
||||
q = torch.randn(1, 128, 2, 96, dtype=torch.bfloat16)
|
||||
with pytest.raises(ValueError, match="diffusion_kv_cache_dtype"):
|
||||
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": method}))
|
||||
runtime.quant_attention.assert_not_called()
|
||||
|
||||
|
||||
def test_packed_metadata_never_reaches_quant_runtime(runtime):
|
||||
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
|
||||
metadata = AttentionMetadata(extra={"kv_cache_dtype": "mxfp8", "cu_seqlens_q": torch.tensor([0, 64, 128])})
|
||||
with pytest.raises(ValueError, match="packed/varlen"):
|
||||
flash().forward_npu(q, q, q, metadata)
|
||||
runtime.quant_attention.assert_not_called()
|
||||
|
||||
|
||||
def make_layer(runtime, monkeypatch, method="mxfp8", skip_steps=_DEFAULT_SKIP_STEPS):
|
||||
layer = object.__new__(layer_mod.Attention)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.attention = flash()
|
||||
layer.attn_backend = flash_attn.FlashAttentionBackend
|
||||
layer._disable_kv_quant = False
|
||||
layer.layer_idx = 3
|
||||
cfg = SimpleNamespace(
|
||||
diffusion_kv_cache_dtype=method,
|
||||
diffusion_kv_cache_skip_step_indices=skip_steps,
|
||||
diffusion_kv_cache_skip_layer_indices=None,
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
layer._init_kv_cache_quantization(cfg)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
|
||||
def test_global_quant_method_is_published_per_forward(runtime, monkeypatch, method):
|
||||
layer = make_layer(runtime, monkeypatch, method=method, skip_steps=None)
|
||||
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == method
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", [None, "auto", "float"])
|
||||
def test_optout_without_quant_policy_keeps_metadata_unchanged(runtime, monkeypatch, method):
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
layer._disable_kv_quant = True
|
||||
layer._init_kv_cache_quantization(
|
||||
SimpleNamespace(
|
||||
diffusion_kv_cache_dtype=method,
|
||||
diffusion_kv_cache_skip_step_indices=None,
|
||||
diffusion_kv_cache_skip_layer_indices=None,
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
)
|
||||
source = video_metadata()
|
||||
assert layer._with_kv_cache_dtype(source) is source
|
||||
|
||||
|
||||
def test_step_skip_is_request_local_and_cross_optout(runtime, monkeypatch):
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
ctx = SimpleNamespace(denoise_step_idx=0)
|
||||
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
|
||||
source = video_metadata()
|
||||
for step in list(range(40)) * 2:
|
||||
ctx.denoise_step_idx = step
|
||||
resolved = layer._with_kv_cache_dtype(source)
|
||||
assert resolved.extra["kv_cache_dtype"] == ("float" if step in (0, 1, 38, 39) else "mxfp8")
|
||||
assert resolved.video_layout is source.video_layout
|
||||
assert source.extra == {"max_seqlen_q": 4096}
|
||||
layer._disable_kv_quant = True
|
||||
assert layer._with_kv_cache_dtype(source).extra["kv_cache_dtype"] == "float"
|
||||
|
||||
|
||||
def test_ring_attention_is_rejected(runtime, monkeypatch):
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
cfg = SimpleNamespace(diffusion_kv_cache_dtype="mxfp8", parallel_config=SimpleNamespace(ring_degree=2))
|
||||
with pytest.raises(ValueError, match="ring"):
|
||||
layer._init_kv_cache_quantization(cfg)
|
||||
|
||||
|
||||
def test_sparse_tail_steps_and_dense_quant_dispatch(runtime, monkeypatch):
|
||||
ctx = SimpleNamespace(denoise_step_idx=49, total_denoise_steps=50)
|
||||
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(rainfusion_attn, "get_forward_context", lambda: ctx)
|
||||
impl = sparse(start_step=2, end_step=2)
|
||||
metadata = video_metadata(kv_cache_dtype="fp8")
|
||||
assert impl._resolve_plan(metadata) is None
|
||||
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
|
||||
impl.forward_npu(q, q, q, metadata)
|
||||
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
|
||||
def test_sparse_quant_uses_public_runtime(runtime, monkeypatch, method):
|
||||
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
|
||||
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
|
||||
q = torch.randn(1, 4224, 2, 64, dtype=torch.bfloat16)
|
||||
out = sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
|
||||
# The sparse path reuses the query tail as the output padding instead of
|
||||
# zero-filling a fresh buffer; rows past the 4096-row valid prefix are
|
||||
# never read back downstream (the model unpads by cu_seqlens), so pin the
|
||||
# passthrough rather than requiring zeros.
|
||||
assert out.shape == q.shape and torch.equal(out[:, 4096:], q[:, 4096:])
|
||||
kwargs = runtime.sparse_attention.call_args.kwargs
|
||||
assert (kwargs["precision"], kwargs["sparse_type"], kwargs["inner_precise"]) == (method, "rf_v3", 4)
|
||||
runtime.quant_attention.assert_not_called()
|
||||
|
||||
|
||||
def test_sparse_custom_mask_stays_dense(runtime):
|
||||
metadata = AttentionMetadata(
|
||||
attn_mask=torch.ones(1, 4096, dtype=torch.bool),
|
||||
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32)),
|
||||
extra={"max_seqlen_q": 4096},
|
||||
)
|
||||
assert sparse()._resolve_plan(metadata) is None
|
||||
|
||||
|
||||
def test_sparse_model_padding_mask_is_equivalent_to_trimming(runtime):
|
||||
metadata = video_metadata()
|
||||
metadata.attn_mask = torch.arange(4224)[None] < 4096
|
||||
assert sparse()._resolve_plan(metadata).used_len == 4096
|
||||
|
||||
|
||||
def test_bsa_operator_failure_never_retries(runtime, monkeypatch):
|
||||
runtime.sparse_attention = Mock(side_effect=RuntimeError("BSA operator failure"))
|
||||
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
|
||||
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
|
||||
with pytest.raises(RuntimeError, match="BSA operator failure"):
|
||||
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype="mxfp4"))
|
||||
runtime.sparse_attention.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,shape,supports_precision,error",
|
||||
[
|
||||
("mxfp8", (1, 4096, 2, 64), True, "sparse mxfp8 is not supported"),
|
||||
("mxfp4", (1, 4096, 2, 64), False, "explicitly support precision"),
|
||||
("fp8", (2, 4096, 2, 64), True, "batch size 1"),
|
||||
("fp8", (1, 4096, 2, 32), True, "power-of-two head dimension"),
|
||||
("mxfp4", (1, 4096, 2, 96), True, "power-of-two head dimension"),
|
||||
],
|
||||
)
|
||||
def test_invalid_sparse_quant_requires_config_change(runtime, monkeypatch, method, shape, supports_precision, error):
|
||||
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
|
||||
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: supports_precision)
|
||||
q = torch.randn(shape, dtype=torch.bfloat16)
|
||||
with pytest.raises(ValueError, match=error):
|
||||
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
|
||||
runtime.sparse_attention.assert_not_called()
|
||||
|
||||
|
||||
def test_layer_selector_requires_index_except_cross_optout(runtime, monkeypatch):
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
layer.layer_idx = None
|
||||
config = SimpleNamespace(
|
||||
diffusion_kv_cache_dtype="mxfp8",
|
||||
diffusion_kv_cache_skip_step_indices=None,
|
||||
diffusion_kv_cache_skip_layer_indices={0, 3},
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
with pytest.raises(ValueError, match="parseable transformer block index"):
|
||||
layer._init_kv_cache_quantization(config)
|
||||
layer._disable_kv_quant = True
|
||||
layer._init_kv_cache_quantization(config)
|
||||
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == "float"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", ["transformer.blocks.3.attn1", "transformer_2.blocks.3.attn1"])
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
|
||||
def test_expert_layer_and_step_skip_remain_sparse(runtime, monkeypatch, prefix, method):
|
||||
def execute(q, k, v, *, precision="bf16", **kwargs):
|
||||
return q
|
||||
|
||||
runtime.sparse_attention = Mock(wraps=execute)
|
||||
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
layer.layer_idx = rainfusion_attn._try_extract_layer_index(prefix)
|
||||
assert layer.layer_idx == 3
|
||||
config = SimpleNamespace(
|
||||
diffusion_kv_cache_dtype=method,
|
||||
diffusion_kv_cache_skip_step_indices={0, 1, 38, 39},
|
||||
diffusion_kv_cache_skip_layer_indices={3},
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
impl = sparse()
|
||||
layer.attention = impl
|
||||
layer.attn_backend = rainfusion_attn.RainFusionAttentionBackend
|
||||
layer._init_kv_cache_quantization(config)
|
||||
ctx = SimpleNamespace(denoise_step_idx=0)
|
||||
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
|
||||
source = video_metadata(kv_cache_dtype="mxfp8")
|
||||
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
|
||||
for layer_idx, step in ((3, 0), (3, 2), (4, 0), (4, 1), (4, 2), (4, 37), (4, 38), (4, 39), (4, 0)):
|
||||
expected = "bf16" if layer_idx == 3 or step in (0, 1, 38, 39) else method
|
||||
layer.layer_idx = layer_idx
|
||||
ctx.denoise_step_idx = step
|
||||
metadata = layer._with_kv_cache_dtype(source)
|
||||
impl.forward_npu(q, q, q, metadata)
|
||||
assert runtime.sparse_attention.call_args.kwargs["precision"] == expected
|
||||
runtime.quant_attention.assert_not_called()
|
||||
assert source.extra["kv_cache_dtype"] == "mxfp8"
|
||||
|
||||
|
||||
def test_custom_attention_initialization_keeps_upstream_optout(monkeypatch):
|
||||
monkeypatch.setattr(layer_mod, "get_current_diffusion_config_or_none", lambda: None)
|
||||
monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kw: object())
|
||||
monkeypatch.setattr(layer_mod, "NoParallelAttention", lambda: object())
|
||||
custom = torch.nn.Identity()
|
||||
layer = layer_mod.Attention(
|
||||
num_heads=2,
|
||||
head_size=64,
|
||||
softmax_scale=0.125,
|
||||
causal=False,
|
||||
custom_attention=custom,
|
||||
skip_sequence_parallel=True,
|
||||
)
|
||||
assert layer.attention is custom and layer._kv_cache_dtype is None
|
||||
|
||||
|
||||
def test_legacy_global_fp8_uses_implicit_bsnd(runtime):
|
||||
q = torch.randn(1, 17, 2, 64, dtype=torch.bfloat16)
|
||||
impl = flash(layout=None)
|
||||
assert impl.forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "fp8"})) is q
|
||||
assert runtime.quant_attention.call_args.kwargs["layout"] == "BSND"
|
||||
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
|
||||
@@ -133,18 +133,17 @@ def test_invalid_ref2va_video_spans_fall_back_to_dense():
|
||||
assert make_impl()._resolve_plan(AttentionMetadata(extra={"max_seqlen_q": 12000}, video_layout=layout)) is None
|
||||
|
||||
|
||||
def test_validate_available_rejects_legacy_mindiesd(monkeypatch):
|
||||
def test_validate_available_accepts_single_video_mindiesd(monkeypatch):
|
||||
mindiesd = types.ModuleType("mindiesd")
|
||||
|
||||
def sparse_attention(query, key, value, **kwargs):
|
||||
return query
|
||||
|
||||
mindiesd.sparse_attention = sparse_attention
|
||||
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
|
||||
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
|
||||
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
|
||||
|
||||
with pytest.raises(ValueError, match="video_spans"):
|
||||
RainFusionAttentionBackend.validate_available()
|
||||
RainFusionAttentionBackend.validate_available()
|
||||
|
||||
|
||||
def test_validate_available_accepts_new_mindiesd(monkeypatch):
|
||||
@@ -153,13 +152,25 @@ def test_validate_available_accepts_new_mindiesd(monkeypatch):
|
||||
def sparse_attention(query, key, value, *, video_spans=None, **kwargs):
|
||||
return query
|
||||
|
||||
mindiesd.sparse_attention = sparse_attention
|
||||
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
|
||||
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
|
||||
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
|
||||
|
||||
RainFusionAttentionBackend.validate_available()
|
||||
|
||||
|
||||
def test_prefix_kv_slicing_contract_is_pinned():
|
||||
# Regression pin for the #5543 packed-padding crash: reverting the flag
|
||||
# would pass CI while reintroducing the ValueError for non-64-aligned
|
||||
# requests. Both halves of the contract must hold together — the impl
|
||||
# trims [real, pad] packed tensors to the valid prefix itself (so it may
|
||||
# advertise prefix slicing) and it never reads attn_mask (so it must keep
|
||||
# refusing one), which is what lets the model skip materializing the
|
||||
# padding mask that _assert_metadata_compatible would reject.
|
||||
assert RainFusionAttentionBackend.supports_prefix_kv_slicing is True
|
||||
assert RainFusionAttentionBackend.supports_attention_mask() is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("grid", [(4, 24, 40), (1, 24, 40)])
|
||||
def test_short_video_stays_dense(grid):
|
||||
assert make_impl()._resolve_plan(make_metadata(grid)) is None
|
||||
@@ -300,22 +311,21 @@ def _fake_mindiesd_module():
|
||||
import types
|
||||
|
||||
fake = types.ModuleType("mindiesd")
|
||||
fake.sparse_attention = lambda *args, **kwargs: None
|
||||
setattr(fake, "sparse_attention", lambda *args, **kwargs: None)
|
||||
return fake
|
||||
|
||||
|
||||
def test_precision_non_bf16_requires_mindiesd_support():
|
||||
"""precision != bf16 against a mindiesd lacking the kwarg must raise RuntimeError."""
|
||||
"""Unsupported sparse precision must fail before any operator executes."""
|
||||
import sys
|
||||
|
||||
impl = make_impl(precision="mix")
|
||||
sys.modules["mindiesd"] = _fake_mindiesd_module()
|
||||
try:
|
||||
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
|
||||
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=False):
|
||||
with pytest.raises(RuntimeError, match="requires MindIE-SD"):
|
||||
impl._forward_sparse_npu(None, None, None, None)
|
||||
finally:
|
||||
sys.modules.pop("mindiesd", None)
|
||||
with pytest.raises(ValueError, match="explicitly support precision"):
|
||||
impl._forward_sparse_npu(
|
||||
None, None, None, RainFusionPlan(prefix_len=0, used_len=8, latent_shape=[2, 2, 2])
|
||||
)
|
||||
|
||||
|
||||
def test_precision_non_bf16_passes_gate_when_supported():
|
||||
@@ -323,8 +333,7 @@ def test_precision_non_bf16_passes_gate_when_supported():
|
||||
import sys
|
||||
|
||||
impl = make_impl(precision="mix")
|
||||
sys.modules["mindiesd"] = _fake_mindiesd_module()
|
||||
try:
|
||||
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
|
||||
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=True):
|
||||
# q/k/v shapes: [B, S, N, D]; plan geometry must match S.
|
||||
q = torch.randn(1, 8, 4, 128)
|
||||
@@ -332,5 +341,3 @@ def test_precision_non_bf16_passes_gate_when_supported():
|
||||
# The gate must pass; the fake mindiesd returns None so no crash.
|
||||
out = impl._forward_sparse_npu(q, q, q, plan)
|
||||
assert out is None
|
||||
finally:
|
||||
sys.modules.pop("mindiesd", None)
|
||||
@@ -325,7 +325,7 @@ def _runner(
|
||||
model_config=SimpleNamespace(max_model_len=max_len),
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_seqs=max_num_seqs * max_rows_per_request,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=cp_size,
|
||||
@@ -403,7 +403,7 @@ def test_initialize_failure_does_not_publish_partial_cache_state(
|
||||
successful_init_cache = model_runner_backend_module.init_kv_cache
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(max_model_len=16),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=1),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=2),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
@@ -445,7 +445,7 @@ def test_initialize_failure_does_not_publish_partial_cache_state(
|
||||
assert backend.block_tables is None
|
||||
assert backend.paged_attention_adapter is None
|
||||
assert backend._kv_cache_layer_adapters["layer-0"].kv_cache == "placeholder-0"
|
||||
assert backend.vllm_config.scheduler_config.max_num_seqs == 1
|
||||
assert backend.vllm_config.scheduler_config.max_num_seqs == 2
|
||||
|
||||
monkeypatch.setattr(model_runner_backend_module, "init_kv_cache", successful_init_cache)
|
||||
backend.initialize_kv_cache(config)
|
||||
|
||||
@@ -58,6 +58,7 @@ def _od_config(**overrides):
|
||||
gpu_memory_utilization=0.9,
|
||||
kv_cache_memory_bytes=None,
|
||||
max_num_seqs=1,
|
||||
diffusion_kv_max_rows_per_request=1,
|
||||
max_num_batched_tokens=64,
|
||||
num_gpus=1,
|
||||
)
|
||||
|
||||
@@ -1,16 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.diffusion_kv.kv_connector import (
|
||||
KVTransferRegistrationError,
|
||||
commit_kv_load,
|
||||
parse_kv_transfer_config,
|
||||
shutdown_kv_connector,
|
||||
wait_for_kv_load,
|
||||
)
|
||||
from vllm_omni.diffusion.sched.base_scheduler import BaseScheduler
|
||||
from vllm_omni.diffusion.vllm_config import create_diffusion_vllm_config
|
||||
@@ -138,3 +145,205 @@ def test_shutdown_kv_connector_is_idempotent() -> None:
|
||||
|
||||
assert ensure_shutdown.call_count == 2
|
||||
scheduler_connector.shutdown.assert_called_once()
|
||||
|
||||
|
||||
def test_commit_aligns_cfg_transport_pages_without_extending_computed_prefix() -> None:
|
||||
requests = tuple(
|
||||
SimpleNamespace(
|
||||
request_id=f"req/{sequence_id}",
|
||||
kv_transfer_params={"num_transfer_tokens": 9},
|
||||
num_tokens=16,
|
||||
num_computed_tokens=0,
|
||||
)
|
||||
for sequence_id in range(2)
|
||||
)
|
||||
manager = SimpleNamespace(
|
||||
get_blocks=lambda _request_id: KVCacheBlocks(([0, 1, 2, 3],)),
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=4))]),
|
||||
)
|
||||
connector = mock.Mock()
|
||||
|
||||
expected = commit_kv_load(connector, manager, requests, [7, 5])
|
||||
|
||||
assert expected == {"req/0", "req/1"}
|
||||
calls = connector.update_state_after_alloc.call_args_list
|
||||
assert [len(call.args[1].blocks[0]) for call in calls] == [3, 3]
|
||||
assert [call.args[2] for call in calls] == [7, 5]
|
||||
assert [request.num_computed_tokens for request in requests] == [7, 5]
|
||||
|
||||
|
||||
def test_wait_without_pending_load_uses_vllm_post_forward_once() -> None:
|
||||
connector = mock.Mock()
|
||||
output = mock.Mock(invalid_block_ids=set())
|
||||
active_connector = mock.Mock(kv_connector=connector)
|
||||
active_connector.post_forward.return_value = output
|
||||
scheduler_output = SimpleNamespace(
|
||||
kv_transfer_request_ids=set(),
|
||||
finished_req_ids={"finished"},
|
||||
kv_finished_request_ids={"finished/diffusion-kv/0", "finished/diffusion-kv/1"},
|
||||
)
|
||||
|
||||
result = wait_for_kv_load(active_connector, scheduler_output, timeout=1.0)
|
||||
|
||||
assert result is output
|
||||
active_connector.pre_forward.assert_called_once_with(scheduler_output)
|
||||
active_connector.post_forward.assert_called_once_with({"finished/diffusion-kv/0", "finished/diffusion-kv/1"})
|
||||
connector.get_finished.assert_not_called()
|
||||
|
||||
|
||||
def test_timeout_returns_partial_completion_without_releasing_pages():
|
||||
active = mock.Mock()
|
||||
active.kv_connector.get_finished.return_value = (set(), {"cfg0"})
|
||||
active.kv_connector.get_block_ids_with_load_errors.return_value = set()
|
||||
active.post_forward.return_value = KVConnectorOutput()
|
||||
scheduled = SimpleNamespace(kv_transfer_request_ids={"cfg0", "cfg1"}, kv_finished_request_ids=set())
|
||||
output = wait_for_kv_load(active, scheduled, timeout=0)
|
||||
assert output.finished_recving == {"cfg0"}
|
||||
active.kv_connector.request_finished.assert_not_called()
|
||||
active.post_forward.assert_called_once_with(set())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cfg_registration():
|
||||
requests = tuple(
|
||||
SimpleNamespace(
|
||||
request_id=f"cfg{i}",
|
||||
status=None,
|
||||
num_tokens=8,
|
||||
kv_transfer_params={"num_transfer_tokens": 4, "transfer_id": "ticket", "do_remote_prefill": True},
|
||||
)
|
||||
for i in range(2)
|
||||
)
|
||||
manager = SimpleNamespace(
|
||||
get_blocks=lambda _: KVCacheBlocks(([0, 1],)),
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=4))]),
|
||||
)
|
||||
return requests, manager
|
||||
|
||||
|
||||
def test_cfg_validation_is_atomic_before_connector_registration(cfg_registration):
|
||||
requests, manager = cfg_registration
|
||||
requests[1].kv_transfer_params["num_transfer_tokens"] = 9
|
||||
connector = mock.Mock()
|
||||
with pytest.raises(KVTransferRegistrationError, match="boundary"):
|
||||
commit_kv_load(connector, manager, requests, [4, 4])
|
||||
connector.update_state_after_alloc.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail_init", [False, True])
|
||||
def test_sp_connector_init_snapshots_topology_and_restores_model_tp(monkeypatch, fail_init):
|
||||
import vllm.distributed.parallel_state as parallel_state
|
||||
|
||||
import vllm_omni.diffusion.diffusion_kv.kv_connector as module
|
||||
import vllm_omni.diffusion.distributed.parallel_state as omni_parallel
|
||||
|
||||
tp = SimpleNamespace(world_size=1, rank_in_group=0)
|
||||
sp = SimpleNamespace(world_size=2, rank_in_group=1)
|
||||
monkeypatch.setattr(parallel_state, "_TP", tp)
|
||||
monkeypatch.setattr(omni_parallel, "get_sp_group", lambda: sp)
|
||||
snapshot: dict[str, int] = {}
|
||||
|
||||
def initialize(*args):
|
||||
snapshot.update(
|
||||
rank=parallel_state.get_tensor_model_parallel_rank(),
|
||||
size=parallel_state.get_tensor_model_parallel_world_size(),
|
||||
)
|
||||
if fail_init:
|
||||
raise RuntimeError("initialization failed")
|
||||
|
||||
monkeypatch.setattr(module, "ensure_kv_transfer_initialized", initialize)
|
||||
config = SimpleNamespace(
|
||||
kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector", engine_id="dit"),
|
||||
parallel_config=SimpleNamespace(prefill_context_parallel_size=2),
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="initialization failed") if fail_init else nullcontext():
|
||||
module.init_worker_kv_connector(config, object())
|
||||
assert parallel_state._TP is tp
|
||||
assert snapshot == {"rank": 1, "size": 2}
|
||||
|
||||
|
||||
def test_partial_mooncake_registration_is_replaced_by_empty_notifications(cfg_registration):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
||||
MooncakeConnector,
|
||||
MooncakeConnectorScheduler,
|
||||
)
|
||||
|
||||
scheduler = object.__new__(MooncakeConnectorScheduler)
|
||||
scheduler.is_kv_producer = False
|
||||
scheduler.is_kv_consumer = True
|
||||
scheduler._reqs_need_recv = {}
|
||||
connector = object.__new__(MooncakeConnector)
|
||||
connector.connector_scheduler = scheduler
|
||||
requests, manager = cfg_registration
|
||||
for request in requests:
|
||||
request.kv_transfer_params.update(remote_engine_id="ar", remote_bootstrap_addr="http://localhost:8998")
|
||||
|
||||
def register(request, blocks, count):
|
||||
scheduler._reqs_need_recv[request.request_id] = (request, blocks)
|
||||
request.kv_transfer_params["do_remote_prefill"] = False
|
||||
if request.request_id == "cfg1":
|
||||
raise RuntimeError("partial registration")
|
||||
|
||||
connector.update_state_after_alloc = register
|
||||
with pytest.raises(KVTransferRegistrationError, match="partial registration"):
|
||||
commit_kv_load(connector, manager, requests, [4, 4])
|
||||
assert set(scheduler._reqs_need_recv) == {"cfg0", "cfg1"}
|
||||
assert all(blocks == [] for _, blocks in scheduler._reqs_need_recv.values())
|
||||
metadata = scheduler.build_connector_meta(None)
|
||||
# Even partial registration failure sends every row in one per-rank batch,
|
||||
# preserving the producer's complete fan-out count for the shared ticket.
|
||||
assert set(metadata.reqs_to_recv["ar"]) == {"cfg0", "cfg1"}
|
||||
assert all(
|
||||
meta.transfer_id == "ticket" and meta.local_block_ids == [] for meta in metadata.reqs_to_recv["ar"].values()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("timeout", [0, -1, float("inf"), "60"])
|
||||
def test_native_transfer_timeout_must_be_positive_finite(timeout):
|
||||
with pytest.raises(ValueError, match="transfer_timeout"):
|
||||
parse_kv_transfer_config(
|
||||
{
|
||||
**KV_TRANSFER_CONFIG,
|
||||
"kv_connector_extra_config": {"transfer_timeout": timeout},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_receive_completion_requires_all_ranks_across_polls():
|
||||
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
|
||||
from vllm_omni.diffusion.sched.interface import CachedRequestData, DiffusionSchedulerOutput
|
||||
|
||||
executor = SimpleNamespace(od_config=SimpleNamespace(num_gpus=2))
|
||||
executor.collective_rpc = mock.Mock(
|
||||
side_effect=[
|
||||
[KVConnectorOutput(finished_recving={"cfg0", "cfg1"}), KVConnectorOutput(finished_recving={"cfg0"})],
|
||||
[KVConnectorOutput(), KVConnectorOutput(finished_recving={"cfg1"})],
|
||||
]
|
||||
)
|
||||
scheduled = DiffusionSchedulerOutput(
|
||||
step_id=0,
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
finished_req_ids=set(),
|
||||
num_running_reqs=0,
|
||||
num_waiting_reqs=0,
|
||||
kv_connector_metadata=object(),
|
||||
kv_transfer_request_ids={"cfg0", "cfg1"},
|
||||
)
|
||||
result = DiffusionExecutor.prepare_kv_for_forward(executor, scheduled)
|
||||
assert result.finished_recving == {"cfg0"}
|
||||
scheduled.kv_transfer_request_ids = set()
|
||||
result = DiffusionExecutor.prepare_kv_for_forward(executor, scheduled)
|
||||
assert result.finished_recving == {"cfg1"}
|
||||
assert not executor._kv_receive_completed_ranks
|
||||
|
||||
|
||||
def test_native_transfer_rejects_sleep_without_changing_legacy_sleep():
|
||||
with pytest.raises(ValueError, match="registered pages must remain mapped"):
|
||||
OmniDiffusionConfig.from_kwargs(
|
||||
diffusion_kv_mode="paged_scheduler",
|
||||
diffusion_kv_max_rows_per_request=2,
|
||||
kv_transfer_config=dict(KV_TRANSFER_CONFIG),
|
||||
enable_sleep_mode=True,
|
||||
)
|
||||
assert OmniDiffusionConfig.from_kwargs(enable_sleep_mode=True).enable_sleep_mode
|
||||
@@ -61,13 +61,14 @@ class _MetadataScheduler(BaseScheduler):
|
||||
|
||||
|
||||
def test_scheduler_copies_opaque_params_to_sequence_requests() -> None:
|
||||
params = {"transfer_id": "xfer-req-0", "do_remote_prefill": True}
|
||||
params = {"transfer_id": "xfer-req-0", "do_remote_prefill": True, "num_transfer_tokens": 3}
|
||||
sequence = DiffusionKVRequest(
|
||||
"req-0/diffusion-kv/0",
|
||||
sequence_id=0,
|
||||
prefix_len=4,
|
||||
target_len=2,
|
||||
seq_len=6,
|
||||
prompt_token_ids=[10, 11, 12, 13],
|
||||
)
|
||||
request = OmniDiffusionRequest(
|
||||
prompt="prompt",
|
||||
@@ -83,6 +84,9 @@ def test_scheduler_copies_opaque_params_to_sequence_requests() -> None:
|
||||
|
||||
assert state.diffusion_kv_requests[0].kv_transfer_params == params
|
||||
assert state.diffusion_kv_requests[0].kv_transfer_params is not params
|
||||
assert sequence.prompt_token_ids == [10, 11, 12]
|
||||
assert sequence.num_prompt_tokens == 3
|
||||
assert sequence.seq_len == 6
|
||||
|
||||
|
||||
def test_diffusion_kv_metadata_uses_native_cache_group_block_ids() -> None:
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Exercise upstream Mooncake completion accounting without device/network I/O."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector as mc
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from vllm_omni.diffusion.diffusion_kv.kv_connector import install_mooncake_cfg_fanout
|
||||
from vllm_omni.worker.mixins import OmniWorkerMixin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def producer(monkeypatch):
|
||||
worker = object.__new__(mc.MooncakeConnectorWorker)
|
||||
worker.shutdown = Mock() # No hardware constructor, sockets or listener thread.
|
||||
worker.is_kv_consumer = False
|
||||
worker.tp_rank, worker.tp_size = 0, 1
|
||||
worker.transfer_topo = SimpleNamespace(handshake_target_ranks=lambda size: list(range(size)))
|
||||
worker.reqs_need_send, worker.finished_sending_reqs = {}, set()
|
||||
for name in (
|
||||
"kv_caches_base_addr",
|
||||
"block_len_per_layer",
|
||||
"kv_block_len_per_layer",
|
||||
"registered_layer_names",
|
||||
"registered_layer_indices",
|
||||
"registered_group_indices",
|
||||
):
|
||||
setattr(worker, name, [])
|
||||
worker._get_transfer_regions = Mock(return_value=[])
|
||||
worker._producer_cache_is_replicated = Mock(return_value=False)
|
||||
worker._encoder = msgspec.msgpack.Encoder()
|
||||
worker._sender_executor = None
|
||||
worker._send_blocks = Mock(return_value=0)
|
||||
worker.xfer_stats = Mock()
|
||||
worker._build_transfer_params = AsyncMock(return_value=([1], [2], [16], [], None))
|
||||
monkeypatch.setattr(mc, "_align_transfer_regions", lambda *_: ([], [], None))
|
||||
monkeypatch.setattr(mc, "_validate_asymmetric_region_lengths", lambda **_: None)
|
||||
monkeypatch.setattr(mc.envs, "VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", 0.05)
|
||||
connector = object.__new__(mc.MooncakeConnector)
|
||||
connector.connector_worker = worker
|
||||
return connector
|
||||
|
||||
|
||||
async def register(worker, ticket):
|
||||
worker.sender_loop = asyncio.get_running_loop()
|
||||
for blocks in ([], [[10]]):
|
||||
meta = mc.MooncakeConnectorMetadata()
|
||||
meta.add_new_req(ticket, blocks, {"transfer_id": ticket}, load_remote_cache=False)
|
||||
await worker.record_send_reqs(meta)
|
||||
return worker.reqs_need_send[ticket]
|
||||
|
||||
|
||||
def packet(rank, ranks, tickets, empty=False):
|
||||
return mc.MooncakeXferMetadata(
|
||||
remote_hostname="localhost",
|
||||
remote_port=1234 + rank,
|
||||
remote_tp_size=ranks,
|
||||
remote_tp_rank=rank,
|
||||
req_blocks={
|
||||
f"{ticket}/{row}": (ticket, [] if empty else [[100 + rank * 10 + row]])
|
||||
for ticket, rows in tickets.items()
|
||||
for row in range(rows)
|
||||
},
|
||||
kv_caches_base_addr=[],
|
||||
block_lens=[],
|
||||
kv_block_lens=[],
|
||||
)
|
||||
|
||||
|
||||
async def send(worker, rank, ranks, tickets, empty=False):
|
||||
sock = SimpleNamespace(send_multipart=AsyncMock())
|
||||
await worker.send_kv_to_decode(str(rank).encode(), sock, packet(rank, ranks, tickets, empty))
|
||||
return [
|
||||
msgspec.msgpack.decode(c.args[0][1], type=mc.MooncakeXferResponse) for c in sock.send_multipart.call_args_list
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("rows", [1, 2])
|
||||
@pytest.mark.parametrize("ranks", [2, 4])
|
||||
@pytest.mark.parametrize("in_flight", [False, True])
|
||||
async def test_source_pages_wait_for_every_rank_and_row(producer, rows, ranks, in_flight):
|
||||
worker = producer.connector_worker
|
||||
entered, release = asyncio.Event(), asyncio.Event()
|
||||
original = worker._build_transfer_params
|
||||
|
||||
async def plan(ready, meta, *regions):
|
||||
if in_flight and meta.remote_tp_rank == ranks - 1:
|
||||
entered.set()
|
||||
await release.wait()
|
||||
return await original(ready, meta, *regions)
|
||||
|
||||
worker._build_transfer_params = plan
|
||||
install_mooncake_cfg_fanout(producer)
|
||||
adapted = worker._build_transfer_params
|
||||
install_mooncake_cfg_fanout(producer)
|
||||
assert worker._build_transfer_params is adapted # Do not multiply twice.
|
||||
ticket = await register(worker, "request")
|
||||
pending = asyncio.create_task(send(worker, ranks - 1, ranks, {"request": rows})) if in_flight else None
|
||||
try:
|
||||
if pending is not None:
|
||||
await asyncio.wait_for(entered.wait(), 2)
|
||||
for rank in range(ranks - 1):
|
||||
await send(worker, rank, ranks, {"request": rows})
|
||||
assert ticket.need_send == ranks * rows
|
||||
assert ticket.sent == (rank + 1) * rows
|
||||
assert worker.reqs_need_send["request"] is ticket
|
||||
finished = await worker.fetch_finished_sending_reqs()
|
||||
scheduler = SimpleNamespace(connector=None, requests={"request": object()}, _free_blocks=Mock())
|
||||
Scheduler._update_from_kv_xfer_finished(scheduler, KVConnectorOutput(finished_sending=finished))
|
||||
scheduler._free_blocks.assert_not_called()
|
||||
release.set()
|
||||
responses = (
|
||||
await asyncio.wait_for(pending, 2) if pending else await send(worker, ranks - 1, ranks, {"request": rows})
|
||||
)
|
||||
assert sum(len(response.ok_reqs or []) for response in responses) == rows
|
||||
assert all(not response.err_reqs for response in responses)
|
||||
assert "request" not in worker.reqs_need_send
|
||||
finished = await worker.fetch_finished_sending_reqs()
|
||||
assert finished == {"request"}
|
||||
Scheduler._update_from_kv_xfer_finished(scheduler, KVConnectorOutput(finished_sending=finished))
|
||||
scheduler._free_blocks.assert_called_once()
|
||||
assert not await worker.fetch_finished_sending_reqs()
|
||||
finally:
|
||||
release.set()
|
||||
if pending is not None:
|
||||
await asyncio.gather(pending, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("empty", [False, True])
|
||||
@pytest.mark.parametrize("ranks", [1, 2])
|
||||
async def test_mixed_tickets_and_empty_block_notifications(producer, empty, ranks):
|
||||
worker = producer.connector_worker
|
||||
if empty:
|
||||
worker._build_transfer_params.return_value = ([], [], [], [], None)
|
||||
install_mooncake_cfg_fanout(producer)
|
||||
single, cfg = await register(worker, "single"), await register(worker, "cfg")
|
||||
await asyncio.gather(*(send(worker, rank, ranks, {"single": 1, "cfg": 2}, empty) for rank in range(ranks)))
|
||||
assert (single.need_send, single.sent) == (ranks, ranks)
|
||||
assert (cfg.need_send, cfg.sent) == (2 * ranks, 2 * ranks)
|
||||
assert await worker.fetch_finished_sending_reqs() == {"single", "cfg"}
|
||||
assert not worker.reqs_need_send
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["row_count", "transport"])
|
||||
async def test_failed_rank_does_not_report_source_complete(producer, failure):
|
||||
worker = producer.connector_worker
|
||||
install_mooncake_cfg_fanout(producer)
|
||||
ticket = await register(worker, "request")
|
||||
await send(worker, 0, 2, {"request": 2})
|
||||
if failure == "transport":
|
||||
worker._send_blocks.return_value = -1
|
||||
responses = await send(worker, 1, 2, {"request": 1 if failure == "row_count" else 2})
|
||||
assert any(response.err_reqs for response in responses)
|
||||
assert ticket.sent == 2 and ticket.need_send == 4 and ticket.sending == 0
|
||||
assert worker.reqs_need_send["request"] is ticket
|
||||
assert not await worker.fetch_finished_sending_reqs()
|
||||
# Preserve upstream timeout cleanup after all in-flight writes have stopped.
|
||||
ticket.expire_time = 0
|
||||
assert await worker.fetch_finished_sending_reqs() == {"request"}
|
||||
assert not worker.reqs_need_send
|
||||
|
||||
|
||||
def test_worker_installs_adapter_after_native_initialization(producer, monkeypatch):
|
||||
import vllm.distributed.kv_transfer as kv_transfer
|
||||
|
||||
worker = producer.connector_worker
|
||||
original = worker._build_transfer_params
|
||||
|
||||
class UpstreamWorker:
|
||||
def initialize_from_config(self, config):
|
||||
assert config == "config"
|
||||
assert worker._build_transfer_params is original
|
||||
|
||||
class OmniWorker(OmniWorkerMixin, UpstreamWorker):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(kv_transfer, "has_kv_transfer_group", lambda: True)
|
||||
monkeypatch.setattr(kv_transfer, "get_kv_transfer_group", lambda: producer)
|
||||
object.__new__(OmniWorker).initialize_from_config("config")
|
||||
assert worker._build_transfer_params is not original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", ["other_connector", "scheduler", "consumer"])
|
||||
def test_adapter_leaves_other_roles_untouched(producer, role):
|
||||
worker = producer.connector_worker
|
||||
original = worker._build_transfer_params
|
||||
if role == "scheduler":
|
||||
producer.connector_worker = None
|
||||
elif role == "consumer":
|
||||
worker.is_kv_consumer = True
|
||||
install_mooncake_cfg_fanout(object() if role == "other_connector" else producer)
|
||||
assert worker._build_transfer_params is original
|
||||
@@ -94,7 +94,7 @@ class _FakeAttentionGroup:
|
||||
return self.builder
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@dataclass(slots=True)
|
||||
class _FakeNativeMetadata:
|
||||
build_id: int
|
||||
causal: bool
|
||||
@@ -102,6 +102,7 @@ class _FakeNativeMetadata:
|
||||
query_start_loc_cpu: torch.Tensor
|
||||
positions: torch.Tensor
|
||||
slot_mappings: torch.Tensor
|
||||
max_num_splits: int = 0
|
||||
|
||||
|
||||
class _FakeLayer:
|
||||
@@ -288,6 +289,8 @@ def test_omni_paged_backend_consumes_context_and_restores_diffusion_shape(
|
||||
assert output.shape == query.shape
|
||||
assert torch.equal(output, query)
|
||||
assert layer.calls[0][0].shape == (5, 2, 4)
|
||||
assert layer.calls[0][1] is None
|
||||
assert layer.calls[0][2] is None
|
||||
assert layer.native_events == ["update", "forward"]
|
||||
assert layer.calls[0][3] is events[0][2]
|
||||
|
||||
@@ -440,9 +443,7 @@ def test_platform_prewrite_updates_once_then_piecewise_reads_cache(monkeypatch:
|
||||
|
||||
def test_omni_paged_backend_runs_hunyuan_piecewise_segments(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
adapter, _, layer, events = _make_adapter(monkeypatch)
|
||||
# Exercise the Ascend-only homogeneous batch path. CUDA keeps its native
|
||||
# output-buffer contract and is covered by the GPU adapter tests.
|
||||
monkeypatch.setattr(adapter_module.current_omni_platform, "is_npu", lambda: True)
|
||||
layer.impl.vllm_flash_attn_version = 2
|
||||
batch = adapter.prepare_batch(
|
||||
[
|
||||
DiffusionPagedAttentionRow(
|
||||
@@ -499,10 +500,10 @@ def test_omni_paged_backend_runs_hunyuan_piecewise_segments(monkeypatch: pytest.
|
||||
assert torch.equal(layer.updates[0][0], key.reshape(12, 2, 4))
|
||||
assert torch.equal(layer.updates[0][1], value.reshape(12, 2, 4))
|
||||
assert [call[0].shape[0] for call in layer.calls] == [4, 6, 2]
|
||||
assert [call[1].shape[0] for call in layer.calls] == [4, 6, 2]
|
||||
assert [call[2].shape[0] for call in layer.calls] == [4, 6, 2]
|
||||
assert all(call[1] is None and call[2] is None for call in layer.calls)
|
||||
assert all(call[3] is event[2] for call, event in zip(layer.calls, events[1:], strict=True))
|
||||
assert [metadata.build_id for metadata in context.piecewise_native_metadata] == [1, 2, 3]
|
||||
assert [metadata.max_num_splits for metadata in context.piecewise_native_metadata] == [1, 1, 1]
|
||||
|
||||
# The first metadata build is the normal whole-query path. Piecewise calls
|
||||
# then use causal [3, 5), full [5, 8), and causal [8, 9) segments.
|
||||
@@ -539,6 +540,51 @@ def test_omni_paged_backend_runs_hunyuan_piecewise_segments(monkeypatch: pytest.
|
||||
]
|
||||
|
||||
|
||||
def test_homogeneous_piecewise_packs_strided_query_only_per_segment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
adapter, _, _, _ = _make_adapter(monkeypatch)
|
||||
batch = adapter.prepare_batch(
|
||||
[
|
||||
DiffusionPagedAttentionRow(
|
||||
request_id="req-0",
|
||||
sequence_id=0,
|
||||
query_len=6,
|
||||
seq_len=9,
|
||||
kv_start_pos=3,
|
||||
),
|
||||
DiffusionPagedAttentionRow(
|
||||
request_id="req-1",
|
||||
sequence_id=1,
|
||||
query_len=6,
|
||||
seq_len=9,
|
||||
kv_start_pos=3,
|
||||
),
|
||||
]
|
||||
)
|
||||
packed_qkv = torch.randn(2, 6, 3, 2, 4)
|
||||
query, key, value = packed_qkv.unbind(dim=2)
|
||||
assert not query.is_contiguous()
|
||||
metadata = SimpleNamespace(
|
||||
attn_mask=None,
|
||||
full_attn_spans=[[(5, 8)], [(5, 8)]],
|
||||
query_ranges=None,
|
||||
extra={},
|
||||
)
|
||||
|
||||
with adapter.activate(batch):
|
||||
context = adapter.prepare_layer_context(
|
||||
"layer-0",
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
omni_attn_metadata=metadata,
|
||||
)
|
||||
|
||||
assert not context.query.is_contiguous()
|
||||
assert context.query.data_ptr() == query.data_ptr()
|
||||
|
||||
|
||||
def test_piecewise_metadata_snapshots_reused_native_scheduler_buffer(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
adapter, _, _, _ = _make_adapter(monkeypatch)
|
||||
shared_scheduler_metadata = torch.zeros(1, dtype=torch.int32)
|
||||
@@ -1232,7 +1278,7 @@ def test_layer_adapter_accepts_platform_native_backend_and_uses_rank_local_heads
|
||||
ulysses_degree=2,
|
||||
)
|
||||
|
||||
assert selected_backends == [(adapter_module.AttentionBackendEnum.FLASH_ATTN, {}, 2)]
|
||||
assert selected_backends == [(adapter_module.AttentionBackendEnum.FLASH_ATTN, {}, 1)]
|
||||
assert specialized_backends == [(native_backend, 2)]
|
||||
assert config.attention_config.backend is None
|
||||
assert config.attention_config.backend_per_kind is original_backend_per_kind
|
||||
|
||||
@@ -43,8 +43,7 @@ class _SmokeDiffusionAttention(nn.Module):
|
||||
self.softmax_scale = _HEAD_SIZE**-0.5
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for native paged attention")
|
||||
def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> None:
|
||||
def _native_adapter(num_rows: int = 1):
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
vllm_config = VllmConfig(
|
||||
cache_config=CacheConfig(
|
||||
@@ -89,8 +88,8 @@ def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> N
|
||||
|
||||
canonical_spec = native_layer.spec
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[build_kv_cache_tensor(canonical_spec, 4, [_LAYER_NAME])],
|
||||
num_blocks=4 * num_rows,
|
||||
kv_cache_tensors=[build_kv_cache_tensor(canonical_spec, 4 * num_rows, [_LAYER_NAME])],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
layer_names=[_LAYER_NAME],
|
||||
@@ -120,14 +119,15 @@ def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> N
|
||||
)
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[_BLOCK_SIZE],
|
||||
max_num_reqs=1,
|
||||
max_num_batched_tokens=32,
|
||||
max_num_reqs=num_rows,
|
||||
max_num_batched_tokens=32 * num_rows,
|
||||
max_num_blocks_per_group=[2],
|
||||
device=device,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
)
|
||||
|
||||
block_tables.append_block_ids(0, ([3, 1],), overwrite=True)
|
||||
for row in range(num_rows):
|
||||
block_tables.append_block_ids(row, ([4 * row + 3, 4 * row + 1],), overwrite=True)
|
||||
block_tables.apply_staged_writes()
|
||||
adapter = DiffusionPagedAttentionAdapter(
|
||||
vllm_config=vllm_config,
|
||||
@@ -136,11 +136,17 @@ def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> N
|
||||
block_tables=block_tables,
|
||||
attn_groups=attn_groups,
|
||||
layers={_LAYER_NAME: native_layer},
|
||||
resolve_row=lambda _request_id, _sequence_id, _context_id: DiffusionPagedAttentionRowBinding(
|
||||
row_index=0,
|
||||
resolve_row=lambda _request_id, sequence_id, _context_id: DiffusionPagedAttentionRowBinding(
|
||||
row_index=sequence_id,
|
||||
max_seq_len=19,
|
||||
),
|
||||
)
|
||||
return adapter, diffusion_layer, device
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for native paged attention")
|
||||
def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> None:
|
||||
adapter, diffusion_layer, device = _native_adapter()
|
||||
prefix_batch = adapter.prepare_batch(
|
||||
[
|
||||
DiffusionPagedAttentionRow(
|
||||
@@ -258,3 +264,48 @@ def test_adapter_executes_native_paged_attention_on_non_contiguous_blocks() -> N
|
||||
torch.testing.assert_close(prefix_output, prefix_reference, rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(suffix_output, suffix_reference, rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(piecewise_output, piecewise_reference, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for native paged attention")
|
||||
def test_hunyuan_homogeneous_cfg_paged_attention_graph_replay() -> None:
|
||||
"""Real paged kernels, strided QKV, mixed image spans and changed replay inputs."""
|
||||
adapter, layer, device = _native_adapter(num_rows=2)
|
||||
packed = torch.randn(2, 19, 3, _NUM_HEADS, _HEAD_SIZE, device=device, dtype=torch.float16)
|
||||
query, key, value = packed.unbind(dim=2)
|
||||
backend = FlashAttentionImpl(
|
||||
num_heads=_NUM_HEADS, head_size=_HEAD_SIZE, softmax_scale=layer.softmax_scale, num_kv_heads=_NUM_HEADS
|
||||
)
|
||||
batch = adapter.prepare_batch(
|
||||
[DiffusionPagedAttentionRow(request_id="cfg", sequence_id=i, query_len=19, seq_len=19) for i in range(2)]
|
||||
)
|
||||
metadata = AttentionMetadata(full_attn_spans=[[(5, 10)], [(5, 10)]])
|
||||
with adapter.activate(batch):
|
||||
context = adapter.prepare_layer_context(_LAYER_NAME, query, key, value, omni_attn_metadata=metadata)
|
||||
assert context.piecewise_plan.homogeneous_batch_shape == (2, 19)
|
||||
warmup = torch.cuda.Stream()
|
||||
warmup.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(warmup):
|
||||
for _ in range(3):
|
||||
backend.forward_paged(context)
|
||||
torch.cuda.current_stream().wait_stream(warmup)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = backend.forward_paged(context)
|
||||
output_ptr = captured.data_ptr()
|
||||
mask = torch.ones(19, 19, device=device, dtype=torch.bool).tril()
|
||||
mask[5:10, 5:10] = True
|
||||
for _ in range(2):
|
||||
packed.copy_(torch.randn_like(packed))
|
||||
graph.replay()
|
||||
actual = captured.clone()
|
||||
eager = backend.forward_paged(context)
|
||||
reference = F.scaled_dot_product_attention(
|
||||
query.transpose(1, 2),
|
||||
key.transpose(1, 2),
|
||||
value.transpose(1, 2),
|
||||
attn_mask=mask,
|
||||
scale=layer.softmax_scale,
|
||||
).transpose(1, 2)
|
||||
torch.testing.assert_close(actual.reshape_as(reference), reference, rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(actual, eager, rtol=2e-2, atol=2e-2)
|
||||
assert captured.data_ptr() == output_ptr
|
||||
@@ -176,10 +176,11 @@ def test_worker_honors_explicit_kv_memory_budget(monkeypatch) -> None:
|
||||
profile_request = object()
|
||||
|
||||
assert worker.determine_available_kv_memory(profile_request) == [4096]
|
||||
worker.model_runner.profile_run.assert_called_once_with(profile_request)
|
||||
worker.model_runner.profile_run.assert_not_called()
|
||||
|
||||
|
||||
def test_worker_treats_zero_kv_memory_budget_as_profiled_auto_sizing(monkeypatch) -> None:
|
||||
monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", SimpleNamespace(empty_cache=Mock()))
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.rank = 0
|
||||
worker.init_snapshot = SimpleNamespace(free_memory=900)
|
||||
@@ -209,6 +210,7 @@ def test_worker_treats_zero_kv_memory_budget_as_profiled_auto_sizing(monkeypatch
|
||||
|
||||
@pytest.mark.parametrize("non_kv_cache_memory", [750, 800])
|
||||
def test_worker_rejects_non_positive_profiled_kv_memory(monkeypatch, non_kv_cache_memory: int) -> None:
|
||||
monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", SimpleNamespace(empty_cache=Mock()))
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.rank = 0
|
||||
worker.init_snapshot = SimpleNamespace(free_memory=900)
|
||||
@@ -245,6 +247,7 @@ def test_worker_rejects_non_positive_profiled_kv_memory(monkeypatch, non_kv_cach
|
||||
|
||||
|
||||
def test_worker_profiles_activation_headroom_instead_of_current_residency(monkeypatch) -> None:
|
||||
monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", SimpleNamespace(empty_cache=Mock()))
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.rank = 0
|
||||
worker.init_snapshot = SimpleNamespace(free_memory=1200)
|
||||
@@ -310,3 +313,31 @@ def test_rank_probe_gathers_local_failure_before_raising(monkeypatch) -> None:
|
||||
)
|
||||
|
||||
assert gathered == [(False, "ValueError: local failure")]
|
||||
|
||||
|
||||
def test_hunyuan_native_layer_identity_matches_ar_spec(monkeypatch):
|
||||
import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_image3_transformer as hy3
|
||||
|
||||
monkeypatch.setattr(hy3, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(hy3, "get_sequence_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(hy3, "get_allgather_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(hy3, "get_sequence_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(hy3, "QKVParallelLinear", lambda **_: nn.Identity())
|
||||
monkeypatch.setattr(hy3, "RowParallelLinear", lambda **_: nn.Identity())
|
||||
monkeypatch.setattr(hy3, "get_rope", lambda **_: nn.Identity())
|
||||
|
||||
# Only backend construction is fake; execute both HY3 constructors and the
|
||||
# real runner spec-discovery path, including the actual Attention prefix.
|
||||
def attention(**kwargs):
|
||||
return _attention(enabled=kwargs.get("paged_kv_cache_role") is not None, prefix=kwargs["prefix"])
|
||||
|
||||
monkeypatch.setattr(hy3, "Attention", attention)
|
||||
model = hy3.HunYuanAttention(
|
||||
config=SimpleNamespace(num_key_value_heads=2, attention_head_dim=8),
|
||||
hidden_size=16,
|
||||
num_heads=2,
|
||||
num_kv_heads=2,
|
||||
prefix="layers.3.self_attn",
|
||||
)
|
||||
runner = _runner(model.image_attn.attn)
|
||||
assert set(runner.get_kv_cache_spec()) == {"model.layers.3.self_attn.attn"}
|
||||
@@ -461,3 +461,46 @@ def test_ring_p2p_parity(world_size: int, dtype: torch.dtype):
|
||||
device_kind="cuda",
|
||||
master_port=29501,
|
||||
)
|
||||
|
||||
|
||||
def _run_fused_qkv_matches_three_4d(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
device_kind: DeviceKind,
|
||||
master_port: int,
|
||||
) -> None:
|
||||
"""LingBot's fused (B, S/N, 3, H, D) exchange must equal three per-tensor 4D exchanges."""
|
||||
from vllm_omni.diffusion.distributed.comm import all_to_all_5D
|
||||
|
||||
device = _worker_device(local_rank, device_kind)
|
||||
if device_kind == "cuda":
|
||||
current_omni_platform.set_device(device)
|
||||
_init_worker(local_rank, world_size, master_port, device_kind)
|
||||
initialize_model_parallel(ulysses_degree=world_size)
|
||||
sp_group = get_sp_group().ulysses_group
|
||||
try:
|
||||
batch, seq_per_rank, heads, head_size = 1, 6, 4 * world_size, 8
|
||||
# Distinguishable values: rank, q/k/v index, position, head, feature.
|
||||
base = torch.arange(seq_per_rank * heads * head_size, dtype=torch.float32, device=device).reshape(
|
||||
batch, seq_per_rank, heads, head_size
|
||||
)
|
||||
tensors = [base + 1000.0 * local_rank + 100000.0 * which for which in range(3)]
|
||||
separate = [SeqAllToAll4D.apply(sp_group, t, 2, 1, False) for t in tensors]
|
||||
fused = all_to_all_5D(torch.stack(tensors, dim=2), scatter_idx=3, gather_idx=1, group=sp_group)
|
||||
assert fused.shape == (batch, seq_per_rank * world_size, 3, heads // world_size, head_size)
|
||||
for which, expected in enumerate(separate):
|
||||
assert torch.equal(fused.unbind(2)[which], expected), f"q/k/v index {which} differs on rank {local_rank}"
|
||||
finally:
|
||||
destroy_distributed_env()
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_fused_qkv_all_to_all_matches_three_4d_exchanges_cpu(world_size: int) -> None:
|
||||
torch.multiprocessing.spawn(
|
||||
_run_fused_qkv_matches_three_4d,
|
||||
args=(world_size, "cpu", 29613),
|
||||
nprocs=world_size,
|
||||
)
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SP subgroup construction in parallel_state.py."""
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for SP subgroup construction for diffusion parallel state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -8,12 +8,55 @@ from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import vllm.distributed.parallel_state as vllm_parallel_state
|
||||
|
||||
from tests.helpers.runtime import get_distributed_init_method
|
||||
from vllm_omni.diffusion.distributed import parallel_state as omni_parallel_state
|
||||
from vllm_omni.diffusion.distributed.parallel_state import RankGenerator, set_seq_parallel_pg
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def ensure_parallel_state_initialized():
|
||||
"""Ensure torch distributed is initialized and that all world / count vars are set."""
|
||||
assert torch.distributed.is_initialized()
|
||||
assert omni_parallel_state._WORLD is not None
|
||||
assert vllm_parallel_state._WORLD is not None
|
||||
assert vllm_parallel_state._NODE_COUNT is not None
|
||||
|
||||
|
||||
def ensure_parallel_state_not_initialized():
|
||||
"""Ensure torch distributed is not initialized and that all world / count vars are unset."""
|
||||
assert not torch.distributed.is_initialized()
|
||||
assert omni_parallel_state._WORLD is None
|
||||
assert vllm_parallel_state._WORLD is None
|
||||
assert vllm_parallel_state._NODE_COUNT is None
|
||||
|
||||
|
||||
def test_omni_manages_vllm_distributed_state(monkeypatch):
|
||||
"""Verify Omni initializes and tears down vLLM's distributed state properly.
|
||||
|
||||
This is a regression test for ensuring vLLM Omni also sets up vLLM's vars correctly
|
||||
to avoid potential misalignment in cases where Omni uses vLLM's native coordinator,
|
||||
e.g., MoE + diffusion.
|
||||
"""
|
||||
ensure_parallel_state_not_initialized()
|
||||
monkeypatch.setattr(omni_parallel_state.current_omni_platform, "get_device_count", lambda: 1)
|
||||
monkeypatch.setattr(omni_parallel_state.current_omni_platform, "set_device", lambda _device: None)
|
||||
omni_parallel_state.init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
local_rank=0,
|
||||
distributed_init_method=get_distributed_init_method(),
|
||||
backend="gloo",
|
||||
)
|
||||
try:
|
||||
ensure_parallel_state_initialized()
|
||||
finally:
|
||||
omni_parallel_state.destroy_distributed_env()
|
||||
ensure_parallel_state_not_initialized()
|
||||
|
||||
|
||||
def _fake_new_group_factory(created_groups: list[SimpleNamespace]):
|
||||
def _fake_new_group(ranks, *args, **kwargs):
|
||||
group = SimpleNamespace(ranks=list(ranks))
|
||||
|
||||
@@ -356,6 +356,94 @@ def test_attention_wrapper_rejects_spatial_extent_change(monkeypatch: pytest.Mon
|
||||
wan_spatial_shard._SPATIAL_SHARD_CONTEXT.reset(token)
|
||||
|
||||
|
||||
def _reference_conv_input(conv, x, cache_x):
|
||||
"""The conv input the previous implementation built: cat, F.pad, then halo_exchange."""
|
||||
padding = list(conv._padding)
|
||||
if cache_x is not None and padding[4] > 0:
|
||||
x = torch.cat([cache_x, x], dim=2)
|
||||
padding[4] -= cache_x.shape[2]
|
||||
x = torch.nn.functional.pad(x, padding)
|
||||
x_padded, _, _ = wan_spatial_shard.halo_exchange(
|
||||
x, group=object(), halo_size=conv.halo_size, split_dim=conv.split_dim
|
||||
)
|
||||
return x_padded, x.shape[conv.split_tensor_dim]
|
||||
|
||||
|
||||
def _dist_conv(split_dim, kernel=(3, 3, 3), padding=(1, 1, 1)):
|
||||
from diffusers.models.autoencoders.autoencoder_kl_wan import WanCausalConv3d
|
||||
|
||||
torch.manual_seed(0)
|
||||
source = WanCausalConv3d(3, 4, kernel, padding=padding)
|
||||
return source, wan_spatial_shard.WanDistCausalConv3d(source, object(), split_dim=split_dim)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("split_dim", ["width", "height"])
|
||||
@pytest.mark.parametrize("with_cache", [False, True])
|
||||
def test_single_pass_conv_input_matches_cat_pad_halo_on_one_rank(monkeypatch, split_dim, with_cache):
|
||||
"""One rank, halo-bearing kernel: same bytes as the previous cat -> F.pad -> halo_exchange path."""
|
||||
monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 1))
|
||||
_, conv = _dist_conv(split_dim)
|
||||
x = torch.randn(1, 3, 2, 6, 8)
|
||||
cache_x = torch.randn(1, 3, 2, 6, 8) if with_cache else None
|
||||
|
||||
expected, expected_extent = _reference_conv_input(conv, x, cache_x)
|
||||
got, got_extent = conv._assemble_input(x, cache_x)
|
||||
|
||||
assert got.shape == expected.shape and got_extent == expected_extent
|
||||
assert torch.equal(got, expected)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_single_pass_conv_rejects_cache_longer_than_causal_padding():
|
||||
_, conv = _dist_conv("width")
|
||||
with pytest.raises(AssertionError, match="temporal cache exceeds"):
|
||||
conv._assemble_input(torch.randn(1, 3, 2, 6, 8), torch.randn(1, 3, 3, 6, 8))
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("split_dim", ["width", "height"])
|
||||
def test_single_pass_conv_without_a_halo_matches_the_stock_causal_conv(monkeypatch, split_dim):
|
||||
"""A kernel with no spatial extent along the split (the temporal conv) needs no halo and equals the stock conv."""
|
||||
monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 1))
|
||||
source, conv = _dist_conv(split_dim, kernel=(3, 1, 1), padding=(1, 0, 0))
|
||||
x = torch.randn(1, 3, 2, 6, 8)
|
||||
cache_x = torch.randn(1, 3, 2, 6, 8)
|
||||
|
||||
assert torch.equal(conv(x, cache_x), source(x, cache_x))
|
||||
assert torch.equal(conv(x, None), source(x, None))
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("split_dim", ["width", "height"])
|
||||
@pytest.mark.parametrize("rank", [0, 1])
|
||||
@pytest.mark.parametrize("kernel,padding", [((3, 3, 3), (1, 1, 1)), ((3, 1, 1), (0, 0, 0))])
|
||||
def test_single_pass_conv_input_matches_cat_pad_halo_on_two_ranks(monkeypatch, split_dim, rank, kernel, padding):
|
||||
"""Same bytes as cat -> F.pad -> halo_exchange, halos included, on either rank of a pair."""
|
||||
monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (rank, 2))
|
||||
|
||||
def fake_p2p(*, rank, world_size, group, top_row_ref, bottom_row_ref, recv_top_buf, recv_bottom_buf):
|
||||
# A neighbour's halo is whatever it sent; stand in with a marker derived from our own edge so
|
||||
# both the reference and the single-pass path see the same exchange. Global edges stay zero.
|
||||
recv_top_buf.copy_(top_row_ref * 0 + 7.0) if rank > 0 else recv_top_buf.zero_()
|
||||
recv_bottom_buf.copy_(bottom_row_ref * 0 + 9.0) if rank < world_size - 1 else recv_bottom_buf.zero_()
|
||||
|
||||
monkeypatch.setattr(wan_spatial_shard, "_halo_exchange_p2p", fake_p2p)
|
||||
_, conv = _dist_conv(split_dim, kernel=kernel, padding=padding)
|
||||
x = torch.randn(1, 3, 1, 6, 8)
|
||||
cache_x = torch.randn(1, 3, 2, 6, 8)
|
||||
|
||||
expected, expected_extent = _reference_conv_input(conv, x, cache_x)
|
||||
got, got_extent = conv._assemble_input(x, cache_x)
|
||||
|
||||
assert got.shape == expected.shape and got_extent == expected_extent
|
||||
assert torch.equal(got, expected)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_halo_exchange_single_rank_noop(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
@@ -41,13 +41,13 @@ from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from vllm_omni.diffusion.layers.mot.mot_qkv_parallel_linear import (
|
||||
from vllm_omni.diffusion.models.bagel.mot.mot_qkv_parallel_linear import (
|
||||
MoTQKVParallelLinear,
|
||||
)
|
||||
from vllm_omni.diffusion.layers.mot.mot_row_parallel_linear import (
|
||||
from vllm_omni.diffusion.models.bagel.mot.mot_row_parallel_linear import (
|
||||
MoTRowParallelLinear,
|
||||
)
|
||||
from vllm_omni.diffusion.layers.mot.ops.mot_gemm import (
|
||||
from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import (
|
||||
get_best_mot_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# ruff: noqa: N803, E741
|
||||
"""Layer-level correctness & performance test for MoTRMSNorm.
|
||||
|
||||
@@ -20,7 +23,7 @@ import torch
|
||||
from vllm import ir
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
|
||||
from vllm_omni.diffusion.layers.mot.mot_layernorm import MoTRMSNorm
|
||||
from vllm_omni.diffusion.models.bagel.mot.mot_layernorm import MoTRMSNorm
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.gpu]
|
||||
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm_omni.diffusion.layers.custom_op as custom_op
|
||||
from vllm_omni.diffusion.layers.mot.mot_layernorm import MoTRMSNorm
|
||||
from vllm_omni.diffusion.models.bagel.mot.mot_layernorm import MoTRMSNorm
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for the CustomOp platform-dispatch base class."""
|
||||
|
||||
import pytest
|
||||
@@ -39,7 +39,7 @@ def test_forward_cuda_still_requires_an_implementation():
|
||||
|
||||
def test_mot_rmsnorm_dispatches_on_xpu_without_an_override():
|
||||
"""MoTRMSNorm relies on the inherited fallback (regression test)."""
|
||||
from vllm_omni.diffusion.layers.mot.mot_layernorm import MoTRMSNorm
|
||||
from vllm_omni.diffusion.models.bagel.mot.mot_layernorm import MoTRMSNorm
|
||||
|
||||
assert "forward_xpu" not in MoTRMSNorm.__dict__
|
||||
|
||||
|
||||
@@ -173,3 +173,20 @@ def test_fused_qk_norm_rope_min_tokens_resolution(monkeypatch):
|
||||
monkeypatch.setenv(env, bad)
|
||||
with pytest.raises(ValueError, match=env):
|
||||
fused_qk_norm_rope_min_tokens(2048)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available() or not HAS_TRITON, reason="CUDA and Triton required")
|
||||
@pytest.mark.parametrize("interleaved", [False, True])
|
||||
def test_fused_qk_norm_rope_large_storage_offsets(interleaved):
|
||||
"""Only three tokens, but the last row lies beyond signed 32-bit indexing."""
|
||||
from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
|
||||
|
||||
# 4 GiB of backing storage; initialize only the three small visible rows.
|
||||
q = torch.empty_strided((3, 1, 128), (2**30, 128, 1), device="cuda", dtype=torch.bfloat16)
|
||||
q.fill_(1)
|
||||
weight = torch.ones(128, device="cuda", dtype=torch.bfloat16)
|
||||
table = torch.zeros(3, 96, device="cuda", dtype=torch.bfloat16)
|
||||
table[:, :48] = 1
|
||||
actual = fused_qk_norm_rope(q, q, weight, weight, table, _EPS, interleaved=interleaved)
|
||||
for value in actual:
|
||||
torch.testing.assert_close(value, torch.ones_like(value))
|
||||
@@ -0,0 +1,125 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from diffusers.models.embeddings import apply_rotary_emb
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def test_fused_qk_rope_fake_and_cpu_eligibility():
|
||||
from vllm_omni.diffusion.layers.fused_qk_rope import (
|
||||
_fused_qk_rope_fake,
|
||||
fused_qk_rope,
|
||||
fused_qk_rope_supported,
|
||||
)
|
||||
|
||||
q = torch.randn(2, 3, 4, 8, dtype=torch.bfloat16)
|
||||
k = torch.randn_like(q)
|
||||
cos = torch.randn(3, 8)
|
||||
sin = torch.randn_like(cos)
|
||||
|
||||
fake_q, fake_k = _fused_qk_rope_fake(q, k, cos, sin)
|
||||
assert fake_q.shape == q.shape and fake_q.dtype == q.dtype
|
||||
assert fake_k.shape == k.shape and fake_k.dtype == k.dtype
|
||||
assert not fused_qk_rope_supported(q, k, cos, sin)
|
||||
with pytest.raises(ValueError, match="contiguous CUDA BF16"):
|
||||
fused_qk_rope(q, k, cos, sin)
|
||||
|
||||
|
||||
def _cuda_inputs(batch: int, sequence: int, heads: int = 24, head_dim: int = 128):
|
||||
q = torch.randn(batch, sequence, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
k = torch.randn_like(q)
|
||||
# Deliberately do not repeat adjacent table columns. Diffusers consumes
|
||||
# full-width tables, so even and odd outputs may use different values.
|
||||
cos = torch.randn(sequence, head_dim, device="cuda", dtype=torch.float32)
|
||||
sin = torch.randn_like(cos)
|
||||
return q, k, cos, sin
|
||||
|
||||
|
||||
def _reference(q, k, cos, sin):
|
||||
rotary_emb = (cos, sin)
|
||||
return (
|
||||
apply_rotary_emb(q, rotary_emb, sequence_dim=1),
|
||||
apply_rotary_emb(k, rotary_emb, sequence_dim=1),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
@pytest.mark.parametrize("sequence", [512, 4096, 4608])
|
||||
def test_fused_qk_rope_is_bit_exact_at_longcat_production_shapes(sequence):
|
||||
from vllm_omni.diffusion.layers.fused_qk_rope import _launch_fused_qk_rope, fused_qk_rope
|
||||
|
||||
torch.manual_seed(sequence)
|
||||
with torch.inference_mode():
|
||||
q, k, cos, sin = _cuda_inputs(1, sequence)
|
||||
expected = _reference(q, k, cos, sin)
|
||||
for actual in (
|
||||
_launch_fused_qk_rope(q, k, cos, sin),
|
||||
fused_qk_rope(q, k, cos, sin),
|
||||
):
|
||||
assert torch.equal(actual[0], expected[0])
|
||||
assert torch.equal(actual[1], expected[1])
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
def test_fused_qk_rope_is_bit_exact_for_batch_and_full_width_odd_even_tables():
|
||||
from vllm_omni.diffusion.layers.fused_qk_rope import fused_qk_rope
|
||||
|
||||
torch.manual_seed(11)
|
||||
with torch.inference_mode():
|
||||
q, k, cos, sin = _cuda_inputs(2, 17, heads=3)
|
||||
assert not torch.equal(cos[:, ::2], cos[:, 1::2])
|
||||
assert not torch.equal(sin[:, ::2], sin[:, 1::2])
|
||||
expected = _reference(q, k, cos, sin)
|
||||
actual = fused_qk_rope(q, k, cos, sin)
|
||||
|
||||
assert torch.equal(actual[0], expected[0])
|
||||
assert torch.equal(actual[1], expected[1])
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
def test_fused_qk_rope_rejects_noncontiguous_and_non_fp32_tables():
|
||||
from vllm_omni.diffusion.layers.fused_qk_rope import fused_qk_rope_supported
|
||||
|
||||
q, k, cos, sin = _cuda_inputs(1, 8, heads=2)
|
||||
q_noncontiguous = torch.randn(1, 8, 2, 256, device="cuda", dtype=torch.bfloat16)[..., ::2]
|
||||
k_noncontiguous = torch.randn_like(q_noncontiguous)
|
||||
cos_noncontiguous = torch.randn(8, 256, device="cuda")[:, ::2]
|
||||
sin_noncontiguous = torch.randn(8, 256, device="cuda")[:, ::2]
|
||||
with torch.inference_mode():
|
||||
assert fused_qk_rope_supported(q, k, cos, sin)
|
||||
assert not fused_qk_rope_supported(q_noncontiguous, k_noncontiguous, cos, sin)
|
||||
assert not fused_qk_rope_supported(q, k, cos_noncontiguous, sin_noncontiguous)
|
||||
assert not fused_qk_rope_supported(q, k, cos.to(torch.bfloat16), sin.to(torch.bfloat16))
|
||||
|
||||
q_wide = torch.randn(1, 8, 2, 256, device="cuda", dtype=torch.bfloat16)
|
||||
k_wide = torch.randn_like(q_wide)
|
||||
cos_wide = torch.randn(8, 256, device="cuda")
|
||||
sin_wide = torch.randn_like(cos_wide)
|
||||
assert not fused_qk_rope_supported(q_wide, k_wide, cos_wide, sin_wide)
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
def test_fused_qk_rope_custom_op_has_fullgraph_fake():
|
||||
from vllm_omni.diffusion.layers.fused_qk_rope import fused_qk_rope
|
||||
|
||||
torch.manual_seed(29)
|
||||
with torch.inference_mode():
|
||||
q, k, cos, sin = _cuda_inputs(1, 19, heads=3)
|
||||
expected = _reference(q, k, cos, sin)
|
||||
compiled = torch.compile(fused_qk_rope, dynamic=True, fullgraph=True)
|
||||
actual = compiled(q, k, cos, sin)
|
||||
|
||||
assert torch.equal(actual[0], expected[0])
|
||||
assert torch.equal(actual[1], expected[1])
|
||||
@@ -18,6 +18,7 @@ from vllm_omni.diffusion.lora.loader import (
|
||||
WanLoraLoaderMixin,
|
||||
_prepare_lora_delta,
|
||||
_remap_state_dict_keys,
|
||||
_resolve_lora_compute_device,
|
||||
)
|
||||
|
||||
HEAD_DIM = 24
|
||||
@@ -521,3 +522,81 @@ def test_prepare_lora_delta_with_compute_device():
|
||||
assert delta is not None
|
||||
assert delta.shape == (HEAD_DIM, HEAD_DIM)
|
||||
assert len(used_keys) == 2
|
||||
|
||||
|
||||
def test_resolve_lora_compute_device_probes_unusable_accelerator(mocker: MockerFixture):
|
||||
"""cuda:N/npu:N may be reported without a physical device; the probe must reject it."""
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.lora.loader.get_local_device",
|
||||
return_value=torch.device("cuda:999"),
|
||||
)
|
||||
assert _resolve_lora_compute_device() is None
|
||||
|
||||
|
||||
def test_load_lora_into_module_cpu_weights_match_cpu_reference(mocker: MockerFixture):
|
||||
"""Regression test: params on CPU used to force compute_device=None and skip the
|
||||
#7005 accelerator path. With an unusable accelerator, fusion must fall back to
|
||||
CPU and still produce base + lora_B @ lora_A."""
|
||||
pipeline = DummyPipeline(NUM_LAYERS, HEAD_DIM)
|
||||
original_params = {name: param.clone() for name, param in pipeline.transformer.named_parameters()}
|
||||
|
||||
lora_state_dict = make_lora_state_dict_for_module(pipeline.transformer)
|
||||
lora_state_dict = _remap_state_dict_keys(lora_state_dict, [(".to_out.0.", ".to_out.")])
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.lora.loader.get_local_device",
|
||||
return_value=torch.device("cuda:999"),
|
||||
)
|
||||
|
||||
used_keys = pipeline.load_lora_into_module(lora_state_dict, pipeline.transformer)
|
||||
assert len(used_keys) > 0
|
||||
assert not (set(lora_state_dict.keys()) - used_keys)
|
||||
|
||||
for name, param in pipeline.transformer.named_parameters():
|
||||
if not name.endswith(".weight"):
|
||||
continue
|
||||
base_key = name[: -len(".weight")]
|
||||
key_a = f"{base_key}.lora_A.weight"
|
||||
key_b = f"{base_key}.lora_B.weight"
|
||||
if key_a in lora_state_dict and key_b in lora_state_dict:
|
||||
expected = original_params[name] + (
|
||||
lora_state_dict[key_b].to(param.dtype) @ lora_state_dict[key_a].to(param.dtype)
|
||||
)
|
||||
assert_close(param, expected)
|
||||
|
||||
|
||||
def test_load_lora_into_module_forwards_usable_accel_device(mocker: MockerFixture):
|
||||
"""A usable accelerator must be forwarded as compute_device for CPU params.
|
||||
A spy forces the actual math back onto CPU so the test runs without a device."""
|
||||
fake_dev = torch.device("cuda:0")
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.lora.loader._resolve_lora_compute_device",
|
||||
return_value=fake_dev,
|
||||
)
|
||||
|
||||
seen_devices = []
|
||||
original = _prepare_lora_delta
|
||||
|
||||
def spy(state_dict, base_key, *args, **kwargs):
|
||||
seen_devices.append(kwargs.get("compute_device"))
|
||||
kwargs["compute_device"] = None
|
||||
return original(state_dict, base_key, *args, **kwargs)
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.lora.loader._prepare_lora_delta",
|
||||
side_effect=spy,
|
||||
)
|
||||
|
||||
pipeline = DummyPipeline(NUM_LAYERS, HEAD_DIM)
|
||||
original_params = {name: param.clone() for name, param in pipeline.transformer.named_parameters()}
|
||||
|
||||
lora_state_dict = make_lora_state_dict_for_module(pipeline.transformer)
|
||||
lora_state_dict = _remap_state_dict_keys(lora_state_dict, [(".to_out.0.", ".to_out.")])
|
||||
|
||||
used_keys = pipeline.load_lora_into_module(lora_state_dict, pipeline.transformer)
|
||||
|
||||
# Every CPU param with a LoRA delta must have received the accelerator device.
|
||||
assert seen_devices
|
||||
assert all(d == fake_dev for d in seen_devices)
|
||||
assert not (set(lora_state_dict.keys()) - used_keys)
|
||||
assert_params_not_equal(pipeline.transformer, original_params)
|
||||
@@ -1241,6 +1241,169 @@ def test_dlo_allgather_rejects_unvalidated_online_quant_method(monkeypatch):
|
||||
loader.load_model(load_device="cpu")
|
||||
|
||||
|
||||
def test_dlo_allgather_allows_unquantized_host_fallback():
|
||||
"""Host-loaded unquantized fallback layers are plain contiguous bf16 —
|
||||
the same runtime layout DLO already shards on the ordinary path — so the
|
||||
allowlist must not reject them."""
|
||||
from vllm_omni.quantization.int8_config import UnquantizedHostLinearMethod
|
||||
|
||||
model = nn.Module()
|
||||
model.transformer = nn.Linear(2, 2, bias=False)
|
||||
model.transformer.quant_method = object.__new__(UnquantizedHostLinearMethod)
|
||||
|
||||
assert DiffusersPipelineLoader._unsupported_dlo_allgather_online_quant_methods(model) == ()
|
||||
|
||||
|
||||
def test_dlo_load_model_keeps_host_fallback_on_cpu_through_post_load_sweep(monkeypatch, mocker):
|
||||
"""Through load_model(): DLO + online quant must build the model inside
|
||||
load_unquantizable_fallback_on_cpu(), and the over-wide fallback weight
|
||||
must survive the post-load sweep (the _process_weights_after_loading pass
|
||||
plus model.to("cpu")) as the same host tensor it was loaded into — never
|
||||
bounced to the accelerator."""
|
||||
import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod
|
||||
from vllm_omni.quantization import int8_config
|
||||
from vllm_omni.quantization.int8_config import (
|
||||
NPU_QUANT_MATMUL_MAX_OUT_FEATURES,
|
||||
DiffusionInt8Config,
|
||||
NPUInt8OnlineLinearMethod,
|
||||
UnquantizedHostLinearMethod,
|
||||
)
|
||||
|
||||
# Same stand-in as TestHostFallbackLoading: the eager fallback delegates to
|
||||
# UnquantizedLinearMethod, which reads the TP group.
|
||||
mock_group = mocker.Mock()
|
||||
mock_group.rank_in_group = 0
|
||||
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=1)
|
||||
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_rank", return_value=0)
|
||||
mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group)
|
||||
|
||||
od_config = _make_dlo_online_quant_config()
|
||||
loader = DiffusersPipelineLoader(LoadConfig(), od_config)
|
||||
|
||||
out_features = NPU_QUANT_MATMUL_MAX_OUT_FEATURES + 1
|
||||
built_with_ctx: list[bool] = []
|
||||
|
||||
def copy_loader(param, loaded_weight, *args, **kwargs):
|
||||
param.data.copy_(loaded_weight)
|
||||
|
||||
def build_model(*_args, **_kwargs):
|
||||
built_with_ctx.append(int8_config._LOAD_UNQUANTIZABLE_FALLBACK_ON_CPU.get())
|
||||
quant_config = DiffusionInt8Config(is_checkpoint_int8_serialized=False, activation_scheme="dynamic")
|
||||
method = NPUInt8OnlineLinearMethod(quant_config)
|
||||
layer = nn.Module()
|
||||
layer.quant_method = method
|
||||
method.create_weights(
|
||||
layer,
|
||||
input_size_per_partition=8,
|
||||
output_partition_sizes=[out_features],
|
||||
input_size=8,
|
||||
output_size=out_features,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=copy_loader,
|
||||
)
|
||||
model = nn.Module()
|
||||
model.transformer = layer
|
||||
return model
|
||||
|
||||
def load_weights(_model, **_kwargs):
|
||||
layer = _model.transformer
|
||||
loaded = torch.arange(out_features * 8, dtype=torch.float32).reshape(out_features, 8).to(torch.bfloat16)
|
||||
layer.weight.weight_loader(layer.weight, loaded)
|
||||
|
||||
loader._init_from_load_format = build_model # type: ignore[method-assign]
|
||||
loader.load_weights = load_weights # type: ignore[method-assign]
|
||||
loader._apply_skip_softmax_calibration = lambda _model: None # type: ignore[method-assign]
|
||||
monkeypatch.setattr(
|
||||
loader_mod,
|
||||
"build_checkpoint_mmap_plan",
|
||||
lambda *_args, **_kwargs: HostWeightPlanResult(
|
||||
None,
|
||||
"online quantization requires the ordinary loader",
|
||||
),
|
||||
)
|
||||
# Diffusion DiT models have no vLLM Attention layers, so the upstream
|
||||
# layerwise finalize is a no-op here; stub it to keep the real
|
||||
# _process_weights_after_loading sweep CPU-only.
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.reload.layerwise.finalize_layerwise_processing",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
|
||||
model = loader.load_model(load_device="cpu", device=torch.device("cpu"))
|
||||
|
||||
# Construction ran inside load_unquantizable_fallback_on_cpu(), so the
|
||||
# over-wide layer took the host-loading fallback...
|
||||
assert built_with_ctx == [True]
|
||||
assert type(model.transformer.quant_method) is UnquantizedHostLinearMethod
|
||||
# ...and the real post-load sweep skipped it via the fully-loaded flag and
|
||||
# model.to("cpu") left the loaded host tensor untouched.
|
||||
assert model.transformer._already_called_process_weights_after_loading
|
||||
assert model.transformer.weight.device.type == "cpu"
|
||||
expected = torch.arange(out_features * 8, dtype=torch.float32).reshape(out_features, 8).to(torch.bfloat16)
|
||||
assert torch.equal(model.transformer.weight, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("offload_after_quant", "ctx_expected"), [(True, True), (False, False)])
|
||||
def test_hsdp_enters_host_fallback_context_only_when_offloading_after_quant(mocker, offload_after_quant, ctx_expected):
|
||||
"""The HSDP path must apply the same host-fallback bound as the ordinary
|
||||
path: with a quant config it initializes on the accelerator, so the
|
||||
load_unquantizable_fallback_on_cpu() context is what keeps over-wide
|
||||
fallback weights off the device before sharding."""
|
||||
import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod
|
||||
from vllm_omni.diffusion.offloader.module_collector import PipelineModules
|
||||
from vllm_omni.quantization import int8_config
|
||||
|
||||
od_config = SimpleNamespace(
|
||||
dtype=torch.float32,
|
||||
parallel_config=SimpleNamespace(
|
||||
use_hsdp=True,
|
||||
hsdp_replicate_size=1,
|
||||
hsdp_shard_size=2,
|
||||
),
|
||||
quantization_config=None,
|
||||
)
|
||||
loader = DiffusersPipelineLoader(LoadConfig(), od_config)
|
||||
loader.quant_config = object()
|
||||
|
||||
model = nn.Module()
|
||||
model.transformer = nn.Linear(2, 2, bias=False)
|
||||
ctx_seen: list[bool] = []
|
||||
|
||||
def build_model(*_args, **_kwargs):
|
||||
ctx_seen.append(int8_config._LOAD_UNQUANTIZABLE_FALLBACK_ON_CPU.get())
|
||||
return model
|
||||
|
||||
loader._init_from_load_format = build_model # type: ignore[method-assign]
|
||||
loader.load_weights = lambda _model: None # type: ignore[method-assign]
|
||||
loader._process_weights_after_loading = lambda *_args: None # type: ignore[method-assign]
|
||||
mocker.patch.object(
|
||||
loader_mod.ModuleDiscovery,
|
||||
"discover",
|
||||
return_value=PipelineModules(
|
||||
dits=[model.transformer],
|
||||
dit_names=["transformer"],
|
||||
vaes=[],
|
||||
encoders=[],
|
||||
encoder_names=[],
|
||||
resident_modules=[],
|
||||
resident_names=[],
|
||||
),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.quantization.hsdp_fp8.prepare_fp8_layers_for_fsdp",
|
||||
side_effect=lambda _model: None,
|
||||
)
|
||||
mocker.patch.object(
|
||||
loader_mod,
|
||||
"apply_hsdp_to_model",
|
||||
side_effect=lambda *_args, **_kwargs: None,
|
||||
)
|
||||
|
||||
loader._load_model_with_hsdp(torch.device("cpu"), offload_after_quant=offload_after_quant)
|
||||
|
||||
assert ctx_seen == [ctx_expected]
|
||||
|
||||
|
||||
def test_dlo_allgather_online_mxfp8_uses_ordinary_loader(monkeypatch):
|
||||
mxfp8_config = pytest.importorskip("vllm_omni.quantization.mxfp8_config")
|
||||
NPUMxfp8OnlineLinearMethod = mxfp8_config.NPUMxfp8OnlineLinearMethod
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
@@ -0,0 +1,454 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from shutil import rmtree
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from tests.model_tests.diffusion.anima_builder import CHECKPOINT_FILENAME, tiny_anima_builder
|
||||
from vllm_omni.diffusion.attention import selector
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_qwenimage import DistributedAutoencoderKLQwenImage
|
||||
from vllm_omni.diffusion.models.anima import pipeline_anima
|
||||
from vllm_omni.diffusion.models.anima.anima_text_conditioner import AnimaTextConditioner
|
||||
from vllm_omni.diffusion.models.anima.anima_transformer import AnimaTransformer3DModel
|
||||
from vllm_omni.diffusion.models.anima.pipeline_anima import AnimaPipeline
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"offload_args",
|
||||
[
|
||||
{"enable_cpu_offload": True},
|
||||
{"enable_layerwise_offload": True},
|
||||
{"enable_distributed_layerwise_offload": True},
|
||||
{"diffusion_offload_config": {"mode": "module", "components": ["dit"]}},
|
||||
{
|
||||
"diffusion_offload_config": {
|
||||
"mode": "layer",
|
||||
"components": ["dit"],
|
||||
"layer_options": {"dit": {"weight_transfer": "rank-local"}},
|
||||
}
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_native_anima_rejects_offload(offload_args):
|
||||
config = OmniDiffusionConfig(model="anima.safetensors", **offload_args)
|
||||
with pytest.raises(NotImplementedError, match="offload"):
|
||||
AnimaPipeline(od_config=config, device=torch.device("cpu"))
|
||||
|
||||
|
||||
def test_native_anima_initializes_without_offload():
|
||||
config = OmniDiffusionConfig(model="anima.safetensors")
|
||||
AnimaPipeline(od_config=config, device=torch.device("cpu"))
|
||||
|
||||
|
||||
def test_anima_registration() -> None:
|
||||
"""The native Anima pipeline is registered and importable."""
|
||||
from vllm_omni.diffusion.registry import DiffusionModelRegistry
|
||||
|
||||
assert DiffusionModelRegistry._try_load_model_cls("AnimaPipeline") is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_class_name", ["AnimaPipeline", "AnimaModularPipeline"])
|
||||
def test_enrich_config_native_anima_checkpoint(tmp_path: Path, model_class_name: str) -> None:
|
||||
"""Native and reference Anima class names resolve to AnimaPipeline."""
|
||||
dummy_checkpoint = tmp_path / "model.safetensors"
|
||||
dummy_checkpoint.write_text("dummy")
|
||||
|
||||
config = OmniDiffusionConfig(
|
||||
model=str(dummy_checkpoint),
|
||||
model_class_name=model_class_name,
|
||||
)
|
||||
config.enrich_config()
|
||||
|
||||
assert config.diffusion_load_format == "default"
|
||||
assert config.model_class_name == "AnimaPipeline"
|
||||
assert config.diffusers_pipeline_cls is None
|
||||
|
||||
|
||||
def test_native_anima_component_paths_use_custom_pipeline_args() -> None:
|
||||
"""Anima resolves shared and component-specific paths from native args."""
|
||||
pipeline = AnimaPipeline.__new__(AnimaPipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
custom_pipeline_args={
|
||||
"components_path": "/tmp/anima-components",
|
||||
"vae_path": "/tmp/anima-vae",
|
||||
"text_encoder_model": "/tmp/anima-text-encoder",
|
||||
}
|
||||
)
|
||||
|
||||
assert pipeline._component_path("vae", "/tmp/default") == "/tmp/anima-vae"
|
||||
assert pipeline._component_path("text_encoder", "/tmp/default") == "/tmp/anima-text-encoder"
|
||||
assert pipeline._component_path("tokenizer", "/tmp/default") == "/tmp/default"
|
||||
|
||||
|
||||
def test_native_anima_converts_original_cosmos_transformer_keys() -> None:
|
||||
"""Original Cosmos checkpoint names map to native module names."""
|
||||
converted = AnimaPipeline._convert_original_transformer_state_dict(
|
||||
{
|
||||
"net.x_embedder.proj.1.weight": "patch",
|
||||
"net.blocks.0.self_attn.q_proj.weight": "q",
|
||||
"net.blocks.0.self_attn.q_norm.weight": "q_norm",
|
||||
"net.blocks.0.mlp.layer1.weight": "mlp",
|
||||
"net.final_layer.linear.weight": "out",
|
||||
"net.accum_iteration": "drop",
|
||||
}
|
||||
)
|
||||
|
||||
assert converted == {
|
||||
"patch_embed.proj.weight": "patch",
|
||||
"transformer_blocks.0.attn1.to_q.weight": "q",
|
||||
"transformer_blocks.0.attn1.norm_q.weight": "q_norm",
|
||||
"transformer_blocks.0.ff.net.0.proj.weight": "mlp",
|
||||
"proj_out.weight": "out",
|
||||
}
|
||||
|
||||
|
||||
def test_native_anima_resolves_vae_scale_factor_from_loaded_vae() -> None:
|
||||
"""The loaded VAE determines the spatial scale factor."""
|
||||
from vllm_omni.diffusion.models.anima.pipeline_anima import _anima_vae_scale_factor_from_vae
|
||||
|
||||
vae = SimpleNamespace(config=SimpleNamespace(spatial_compression_ratio=16))
|
||||
|
||||
assert _anima_vae_scale_factor_from_vae(vae) == 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_config_file", [False, True])
|
||||
def test_native_anima_loads_synthetic_checkpoint(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, use_config_file: bool
|
||||
) -> None:
|
||||
"""Native modules load exact weights from an Anima safetensors checkpoint."""
|
||||
tiny_transformer_config = {
|
||||
"in_channels": 1,
|
||||
"out_channels": 1,
|
||||
"num_attention_heads": 1,
|
||||
"attention_head_dim": 12,
|
||||
"num_layers": 1,
|
||||
"mlp_ratio": 1.0,
|
||||
"text_embed_dim": 4,
|
||||
"adaln_lora_dim": 3,
|
||||
"max_size": (1, 2, 2),
|
||||
"patch_size": (1, 1, 1),
|
||||
"rope_scale": (1.0, 1.0, 1.0),
|
||||
"concat_padding_mask": True,
|
||||
"extra_pos_embed_type": None,
|
||||
}
|
||||
tiny_text_conditioner_config = {
|
||||
"source_dim": 4,
|
||||
"target_dim": 4,
|
||||
"model_dim": 4,
|
||||
"num_layers": 1,
|
||||
"num_attention_heads": 1,
|
||||
"target_vocab_size": 8,
|
||||
"min_sequence_length": 4,
|
||||
}
|
||||
if use_config_file:
|
||||
(tmp_path / "anima.json").write_text(
|
||||
json.dumps({"transformer": tiny_transformer_config, "text_conditioner": tiny_text_conditioner_config})
|
||||
)
|
||||
else:
|
||||
monkeypatch.setattr(pipeline_anima, "ANIMA_TRANSFORMER_CONFIG", tiny_transformer_config)
|
||||
monkeypatch.setattr(pipeline_anima, "ANIMA_TEXT_CONDITIONER_CONFIG", tiny_text_conditioner_config)
|
||||
|
||||
monkeypatch.setattr(selector, "_cached_get_backend_cls", lambda *_args, **_kwargs: SDPABackend)
|
||||
transformer = AnimaTransformer3DModel(**tiny_transformer_config)
|
||||
text_conditioner = AnimaTextConditioner(**tiny_text_conditioner_config)
|
||||
transformer_state = {name: tensor.detach().clone() for name, tensor in transformer.state_dict().items()}
|
||||
text_conditioner_state = {name: tensor.detach().clone() for name, tensor in text_conditioner.state_dict().items()}
|
||||
checkpoint_state = {
|
||||
**{f"transformer.{name}": tensor for name, tensor in transformer_state.items()},
|
||||
**{f"text_conditioner.{name}": tensor for name, tensor in text_conditioner_state.items()},
|
||||
}
|
||||
|
||||
checkpoint_path = tmp_path / "anima.safetensors"
|
||||
save_file(checkpoint_state, str(checkpoint_path))
|
||||
|
||||
pipeline = AnimaPipeline.__new__(AnimaPipeline)
|
||||
pipeline.od_config = SimpleNamespace(model=str(checkpoint_path), dtype=torch.float32)
|
||||
pipeline.device = torch.device("cpu")
|
||||
|
||||
def assert_loaded(loaded_transformer, loaded_text_conditioner):
|
||||
for name, tensor in transformer_state.items():
|
||||
assert torch.equal(loaded_transformer.state_dict()[name], tensor)
|
||||
for name, tensor in text_conditioner_state.items():
|
||||
assert torch.equal(loaded_text_conditioner.state_dict()[name], tensor)
|
||||
|
||||
loaded_transformer, loaded_text_conditioner = pipeline._load_native_denoiser_components(dict(checkpoint_state))
|
||||
assert_loaded(loaded_transformer, loaded_text_conditioner)
|
||||
|
||||
loaded_transformer, loaded_text_conditioner = pipeline._load_native_denoiser_components()
|
||||
assert_loaded(loaded_transformer, loaded_text_conditioner)
|
||||
|
||||
if use_config_file:
|
||||
tiny_transformer_config["num_layers"] = 2
|
||||
(tmp_path / "anima.json").write_text(
|
||||
json.dumps({"transformer": tiny_transformer_config, "text_conditioner": tiny_text_conditioner_config})
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="Missing key"):
|
||||
pipeline._load_native_denoiser_components()
|
||||
|
||||
|
||||
def test_tiny_anima_generates_images(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Load every saved component and exercise CFG, decoding, and multiple outputs on CPU."""
|
||||
monkeypatch.setattr(selector, "_cached_get_backend_cls", lambda *_args, **_kwargs: SDPABackend)
|
||||
monkeypatch.setattr(current_omni_platform, "is_available", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
DistributedAutoencoderKLQwenImage,
|
||||
"init_distributed",
|
||||
lambda self: setattr(self, "distributed_executor", SimpleNamespace(parallel_size=1)),
|
||||
)
|
||||
model_dir = tiny_anima_builder()
|
||||
try:
|
||||
config = OmniDiffusionConfig(
|
||||
model=str(Path(model_dir) / CHECKPOINT_FILENAME), model_class_name="AnimaPipeline", dtype=torch.float32
|
||||
)
|
||||
pipeline = AnimaPipeline(od_config=config, device=torch.device("cpu"))
|
||||
pipeline.load_weights()
|
||||
pipeline.eval()
|
||||
outputs = []
|
||||
with torch.inference_mode():
|
||||
for _ in range(2):
|
||||
request = OmniDiffusionRequest(
|
||||
prompt="Dummy prompt",
|
||||
request_id="tiny-anima",
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
height=32,
|
||||
width=32,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=4.0,
|
||||
num_outputs_per_prompt=2,
|
||||
generator=torch.Generator(device="cpu").manual_seed(42),
|
||||
),
|
||||
)
|
||||
outputs.append(pipeline(DiffusionRequestBatch([request]))[0].output)
|
||||
assert outputs[0].shape == (2, 3, 32, 32)
|
||||
assert torch.isfinite(outputs[0]).all()
|
||||
assert torch.equal(outputs[0], outputs[1])
|
||||
assert not torch.equal(outputs[0][0], outputs[0][1])
|
||||
finally:
|
||||
rmtree(model_dir)
|
||||
|
||||
|
||||
def test_native_anima_profiler_setup_is_deferred_and_idempotent(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Profiler targets are wrapped once, after deferred components are loaded."""
|
||||
pipeline = AnimaPipeline.__new__(AnimaPipeline)
|
||||
pipeline.od_config = SimpleNamespace(enable_diffusion_pipeline_profiler=True)
|
||||
pipeline._profiler_initialized = False
|
||||
pipeline.vae = SimpleNamespace(decode=lambda: "decoded")
|
||||
pipeline.text_encoder = SimpleNamespace(forward=lambda: "encoded")
|
||||
|
||||
monkeypatch.setattr(current_omni_platform, "is_available", lambda: False)
|
||||
pipeline._setup_profiler()
|
||||
wrapped_decode = pipeline.vae.decode
|
||||
wrapped_text_encoder = pipeline.text_encoder.forward
|
||||
pipeline._setup_profiler()
|
||||
|
||||
assert pipeline.vae.decode is wrapped_decode
|
||||
assert pipeline.text_encoder.forward is wrapped_text_encoder
|
||||
assert pipeline.vae.decode() == "decoded"
|
||||
assert pipeline.text_encoder.forward() == "encoded"
|
||||
assert "AnimaPipeline.vae.decode" in pipeline.stage_durations
|
||||
assert "AnimaPipeline.text_encoder.forward" in pipeline.stage_durations
|
||||
|
||||
|
||||
def _make_anima_forward_probe():
|
||||
pipeline = AnimaPipeline.__new__(AnimaPipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.vae_scale_factor = 8
|
||||
pipeline.transformer = SimpleNamespace(dtype=torch.float32)
|
||||
pipeline.text_encoder = SimpleNamespace(dtype=torch.float32)
|
||||
pipeline._current_timestep = None
|
||||
pipeline._num_timesteps = 0
|
||||
pipeline._guidance_scale = 0.0
|
||||
captured = {}
|
||||
|
||||
def encode_prompt(**_kwargs):
|
||||
return {
|
||||
"qwen_prompt_embeds": torch.zeros(1, 2, 4),
|
||||
"qwen_attention_mask": torch.ones(1, 2),
|
||||
"t5_input_ids": torch.ones(1, 2, dtype=torch.long),
|
||||
"t5_attention_mask": torch.ones(1, 2),
|
||||
"negative_qwen_prompt_embeds": torch.zeros(1, 2, 4),
|
||||
"negative_qwen_attention_mask": torch.ones(1, 2),
|
||||
"negative_t5_input_ids": torch.ones(1, 2, dtype=torch.long),
|
||||
"negative_t5_attention_mask": torch.ones(1, 2),
|
||||
}
|
||||
|
||||
def condition_prompt_embeds(**_kwargs):
|
||||
return torch.zeros(1, 2, 4)
|
||||
|
||||
def prepare_latents(**kwargs):
|
||||
captured["prepare_latents"] = kwargs
|
||||
return torch.zeros(1, 16, 1, kwargs["height"] // 8, kwargs["width"] // 8)
|
||||
|
||||
def diffuse(**kwargs):
|
||||
captured["diffuse"] = kwargs
|
||||
return kwargs["latents"]
|
||||
|
||||
pipeline.encode_prompt = encode_prompt
|
||||
pipeline.condition_prompt_embeds = condition_prompt_embeds
|
||||
pipeline.prepare_latents = prepare_latents
|
||||
pipeline.prepare_timesteps = lambda **_kwargs: (torch.ones(1), 1)
|
||||
pipeline.diffuse = diffuse
|
||||
|
||||
def decode_latents(latents, output_type="pil"):
|
||||
captured["output_type"] = output_type
|
||||
return DiffusionOutput(output=latents)
|
||||
|
||||
pipeline.decode_latents = decode_latents
|
||||
return pipeline, captured
|
||||
|
||||
|
||||
def test_native_anima_forward_uses_official_default_resolution() -> None:
|
||||
"""Forward uses Anima's reference resolution when none is requested."""
|
||||
pipeline, captured = _make_anima_forward_probe()
|
||||
req = OmniDiffusionRequest(
|
||||
prompt="a red cube",
|
||||
sampling_params=OmniDiffusionSamplingParams(),
|
||||
request_id="anima-defaults",
|
||||
)
|
||||
|
||||
pipeline.forward(DiffusionRequestBatch([req]))
|
||||
|
||||
assert captured["prepare_latents"]["height"] == 1024
|
||||
assert captured["prepare_latents"]["width"] == 1024
|
||||
|
||||
|
||||
def test_native_anima_forward_honors_sampling_output_type() -> None:
|
||||
"""Request-level output type controls Anima decoding."""
|
||||
pipeline, captured = _make_anima_forward_probe()
|
||||
req = OmniDiffusionRequest(
|
||||
prompt="a red cube",
|
||||
sampling_params=OmniDiffusionSamplingParams(output_type="latent"),
|
||||
request_id="anima-output-type",
|
||||
)
|
||||
|
||||
outputs = pipeline.forward(DiffusionRequestBatch([req]))
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], DiffusionOutput)
|
||||
assert outputs[0].output.shape == (1, 16, 1, 128, 128)
|
||||
assert captured["output_type"] == "latent"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("request_count", [0, 2])
|
||||
def test_native_anima_forward_rejects_non_single_request_batches(request_count: int) -> None:
|
||||
pipeline, captured = _make_anima_forward_probe()
|
||||
requests = [
|
||||
OmniDiffusionRequest(
|
||||
prompt="a red cube",
|
||||
sampling_params=OmniDiffusionSamplingParams(),
|
||||
request_id=f"anima-{index}",
|
||||
)
|
||||
for index in range(request_count)
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError, match="supports one request per forward"):
|
||||
pipeline.forward(DiffusionRequestBatch(requests))
|
||||
|
||||
assert captured == {}
|
||||
|
||||
|
||||
def test_native_anima_explicit_guidance_scale_drives_cfg_multiplier() -> None:
|
||||
"""An explicit guidance scale controls native classifier-free guidance."""
|
||||
pipeline, captured = _make_anima_forward_probe()
|
||||
req = OmniDiffusionRequest(
|
||||
prompt="a red cube",
|
||||
sampling_params=OmniDiffusionSamplingParams(guidance_scale=5.0),
|
||||
request_id="anima-guidance",
|
||||
)
|
||||
|
||||
pipeline.forward(DiffusionRequestBatch([req]))
|
||||
|
||||
assert captured["diffuse"]["do_true_cfg"] is True
|
||||
assert captured["diffuse"]["true_cfg_scale"] == 5.0
|
||||
|
||||
|
||||
def test_native_anima_true_cfg_scale_overrides_guidance_multiplier() -> None:
|
||||
"""An explicit true-CFG scale overrides the guidance-scale multiplier."""
|
||||
pipeline, captured = _make_anima_forward_probe()
|
||||
req = OmniDiffusionRequest(
|
||||
prompt="a red cube",
|
||||
sampling_params=OmniDiffusionSamplingParams(guidance_scale=5.0, true_cfg_scale=3.0),
|
||||
request_id="anima-true-cfg",
|
||||
)
|
||||
|
||||
pipeline.forward(DiffusionRequestBatch([req]))
|
||||
|
||||
assert captured["diffuse"]["do_true_cfg"] is True
|
||||
assert captured["diffuse"]["true_cfg_scale"] == 3.0
|
||||
|
||||
|
||||
def test_native_anima_cfg_equation() -> None:
|
||||
"""The denoising loop applies the standard true-CFG equation."""
|
||||
pipeline = AnimaPipeline.__new__(AnimaPipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline._interrupt = False
|
||||
|
||||
class MockProgressBar:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
def update(self):
|
||||
pass
|
||||
|
||||
pipeline.progress_bar = lambda **_kwargs: MockProgressBar()
|
||||
|
||||
class MockTransformer:
|
||||
dtype = torch.float32
|
||||
|
||||
def __call__(self, hidden_states, timestep, encoder_hidden_states, padding_mask, return_dict=False):
|
||||
# Return double the encoder_hidden_states to simulate transformer output
|
||||
return (encoder_hidden_states * 2.0,)
|
||||
|
||||
pipeline.transformer = MockTransformer()
|
||||
|
||||
class MockScheduler:
|
||||
def set_begin_index(self, index):
|
||||
pass
|
||||
|
||||
def step(self, noise_pred, t, latents, return_dict=False):
|
||||
# Return the noise_pred itself to inspect it
|
||||
return (noise_pred,)
|
||||
|
||||
pipeline.scheduler = MockScheduler()
|
||||
|
||||
prompt_embeds = torch.tensor([[[2.0, 3.0]]])
|
||||
negative_prompt_embeds = torch.tensor([[[1.0, 2.0]]])
|
||||
latents = torch.zeros(1, 16, 1, 16, 16)
|
||||
padding_mask = torch.zeros(1, 1, 16, 16)
|
||||
timesteps = torch.tensor([500.0])
|
||||
|
||||
# Cond = prompt_embeds * 2.0 = [4.0, 6.0]
|
||||
# Uncond = negative_prompt_embeds * 2.0 = [2.0, 4.0]
|
||||
# Expected noise_pred = uncond + true_cfg_scale * (cond - uncond)
|
||||
# = [2.0, 4.0] + 4.0 * ([4.0, 6.0] - [2.0, 4.0])
|
||||
# = [2.0, 4.0] + 4.0 * [2.0, 2.0]
|
||||
# = [2.0, 4.0] + [8.0, 8.0] = [10.0, 12.0]
|
||||
out_latents = pipeline.diffuse(
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
latents=latents,
|
||||
padding_mask=padding_mask,
|
||||
timesteps=timesteps,
|
||||
do_true_cfg=True,
|
||||
true_cfg_scale=4.0,
|
||||
)
|
||||
|
||||
expected = torch.tensor([[[10.0, 12.0]]])
|
||||
assert torch.allclose(out_latents, expected)
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Focused CPU tests for BAGEL step execution and packed batching."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -18,12 +18,14 @@ from vllm_omni.diffusion.models.bagel.bagel_transformer import Bagel, NaiveCache
|
||||
from vllm_omni.diffusion.models.bagel.pipeline_bagel import (
|
||||
BagelGenParams,
|
||||
BagelPipeline,
|
||||
_bagel_canvas_requested,
|
||||
get_bagel_pre_process_func,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.sched import StepScheduler
|
||||
from vllm_omni.diffusion.worker.input_batch import InputBatch
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -118,7 +120,7 @@ def test_bagel_step_preprocessor_separates_incompatible_cfg_settings(first_extra
|
||||
assert first.batch_compatibility_key != second.batch_compatibility_key
|
||||
|
||||
|
||||
def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
|
||||
def _pre_process_with_stub_checkpoint(tmp_path):
|
||||
(tmp_path / "config.json").write_text(
|
||||
'{"vae_config":{"downsample":8},"latent_patch_size":2,"max_latent_size":32}',
|
||||
encoding="utf-8",
|
||||
@@ -131,9 +133,22 @@ def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
|
||||
'{"weight_map":{"latent_pos_embed.pos_embed":"ema.safetensors"}}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
pre_process = get_bagel_pre_process_func(
|
||||
types.SimpleNamespace(model=str(tmp_path), revision=None, step_execution=True)
|
||||
return get_bagel_pre_process_func(types.SimpleNamespace(model=str(tmp_path), revision=None, step_execution=True))
|
||||
|
||||
|
||||
def _edit_request(source_size, sampling_params, request_id="req"):
|
||||
return OmniDiffusionRequest(
|
||||
prompt={
|
||||
"prompt": "edit this image",
|
||||
"multi_modal_data": {"image": [Image.new("RGB", source_size)]},
|
||||
},
|
||||
sampling_params=sampling_params,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
|
||||
def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
requests = [
|
||||
OmniDiffusionRequest(
|
||||
prompt={
|
||||
@@ -163,6 +178,172 @@ def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
|
||||
assert schedule.num_waiting_reqs == 1
|
||||
|
||||
|
||||
def test_bagel_pre_process_keeps_explicitly_requested_canvas(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = _edit_request((512, 1024), OmniDiffusionSamplingParams(num_inference_steps=2, height=1024, width=1024))
|
||||
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
|
||||
assert _bagel_canvas_requested(request.sampling_params)
|
||||
|
||||
|
||||
def test_bagel_pre_process_aligns_source_derived_canvas_for_size_auto(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = _edit_request(
|
||||
(500, 700),
|
||||
OmniDiffusionSamplingParams(
|
||||
num_inference_steps=2,
|
||||
height=700,
|
||||
width=500,
|
||||
height_not_provided=True,
|
||||
width_not_provided=True,
|
||||
),
|
||||
)
|
||||
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (704, 496)
|
||||
assert not _bagel_canvas_requested(request.sampling_params)
|
||||
|
||||
|
||||
def test_bagel_pre_process_derives_canvas_when_caller_asked_for_none(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = _edit_request((800, 400), OmniDiffusionSamplingParams(num_inference_steps=2))
|
||||
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (400, 800)
|
||||
assert (request.sampling_params.height_not_provided, request.sampling_params.width_not_provided) == (True, True)
|
||||
assert not _bagel_canvas_requested(request.sampling_params)
|
||||
|
||||
|
||||
def test_bagel_pre_process_marks_kv_metadata_canvas_as_requested(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = _edit_request(
|
||||
(500, 700),
|
||||
OmniDiffusionSamplingParams(
|
||||
num_inference_steps=2,
|
||||
height=700,
|
||||
width=500,
|
||||
height_not_provided=True,
|
||||
width_not_provided=True,
|
||||
kv_metadata={"image_shape": (1024, 1024)},
|
||||
),
|
||||
)
|
||||
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
|
||||
assert (request.sampling_params.height_not_provided, request.sampling_params.width_not_provided) == (False, False)
|
||||
assert _bagel_canvas_requested(request.sampling_params)
|
||||
|
||||
|
||||
def test_bagel_pre_process_prefers_explicit_canvas_over_kv_metadata(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = _edit_request(
|
||||
(1280, 720),
|
||||
OmniDiffusionSamplingParams(
|
||||
num_inference_steps=2,
|
||||
height=1024,
|
||||
width=1024,
|
||||
kv_metadata={"image_shape": (576, 1024)},
|
||||
),
|
||||
)
|
||||
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
|
||||
assert _bagel_canvas_requested(request.sampling_params)
|
||||
|
||||
|
||||
def test_bagel_pre_process_floors_unaligned_explicit_canvas_with_warning(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = _edit_request(
|
||||
(1280, 720),
|
||||
OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
|
||||
)
|
||||
|
||||
with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.logger.warning") as warning:
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (688, 992)
|
||||
assert _bagel_canvas_requested(request.sampling_params)
|
||||
warning.assert_called_once()
|
||||
assert "width 1000->992, height 700->688" in warning.call_args.args[-1]
|
||||
|
||||
|
||||
def test_bagel_pre_process_keeps_aligned_explicit_canvas_silent(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = _edit_request(
|
||||
(1280, 720),
|
||||
OmniDiffusionSamplingParams(num_inference_steps=2, height=512, width=1024),
|
||||
)
|
||||
|
||||
with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.logger.warning") as warning:
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (512, 1024)
|
||||
warning.assert_not_called()
|
||||
|
||||
|
||||
def test_bagel_pre_process_rejects_explicit_canvas_over_checkpoint_limit(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = _edit_request(
|
||||
(1280, 720),
|
||||
OmniDiffusionSamplingParams(num_inference_steps=2, height=512, width=2048),
|
||||
)
|
||||
|
||||
with pytest.raises(OmniClientError, match="width=2048 exceeds the BAGEL checkpoint limit of 1024") as excinfo:
|
||||
pre_process(request)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
|
||||
|
||||
def test_bagel_pre_process_aligns_text2img_canvas_too(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = OmniDiffusionRequest(
|
||||
prompt="a cat",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
|
||||
request_id="req",
|
||||
)
|
||||
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (688, 992)
|
||||
|
||||
|
||||
def test_bagel_pre_process_skips_canvas_alignment_for_text_output(tmp_path):
|
||||
pre_process = _pre_process_with_stub_checkpoint(tmp_path)
|
||||
request = OmniDiffusionRequest(
|
||||
prompt={
|
||||
"prompt": "describe this image",
|
||||
"modalities": ["text"],
|
||||
"multi_modal_data": {"image": [Image.new("RGB", (1280, 720))]},
|
||||
},
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
|
||||
request_id="req",
|
||||
)
|
||||
|
||||
pre_process(request)
|
||||
|
||||
assert (request.sampling_params.height, request.sampling_params.width) == (700, 1000)
|
||||
assert request.use_step_execution is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sampling_kwargs", "requested"),
|
||||
[
|
||||
({"height": 1024, "width": 1024}, True),
|
||||
({"height": 1024, "width": 512, "height_not_provided": True, "width_not_provided": True}, False),
|
||||
({}, False),
|
||||
],
|
||||
ids=["explicit_size", "size_auto", "unset"],
|
||||
)
|
||||
def test_bagel_canvas_requested_separates_asked_for_from_derived(sampling_kwargs, requested):
|
||||
assert _bagel_canvas_requested(OmniDiffusionSamplingParams(**sampling_kwargs)) is requested
|
||||
|
||||
|
||||
def _cache(length: int, value: float, num_layers: int = 1) -> NaiveCache:
|
||||
cache = NaiveCache(num_layers)
|
||||
if length:
|
||||
|
||||
@@ -13,11 +13,13 @@ Two groups:
|
||||
CFG handling, and reshape logic can be verified numerically on CPU.
|
||||
"""
|
||||
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3VLConfig
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig, TransformerConfig
|
||||
|
||||
@@ -56,9 +58,17 @@ def mock_dependencies(mocker, monkeypatch):
|
||||
f"{_MODULE}.FlowMatchEulerDiscreteScheduler.from_pretrained",
|
||||
lambda *a, **k: mock_scheduler,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mllm_loader = mocker.patch(
|
||||
f"{_MODULE}.Qwen3VLForConditionalGeneration.from_pretrained",
|
||||
lambda *a, **k: mllm_wrapper,
|
||||
return_value=mllm_wrapper,
|
||||
)
|
||||
mllm_config_loader = mocker.patch(
|
||||
f"{_MODULE}.Qwen3VLConfig.from_pretrained",
|
||||
return_value=Qwen3VLConfig(),
|
||||
)
|
||||
mllm_builder = mocker.patch(
|
||||
f"{_MODULE}.create_transformers_model_with_vllm_linears",
|
||||
return_value=nn.Linear(1, 1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
f"{_MODULE}.Qwen3VLProcessor.from_pretrained",
|
||||
@@ -74,12 +84,17 @@ def mock_dependencies(mocker, monkeypatch):
|
||||
mock_transformer_cls.return_value = mock_transformer_instance
|
||||
monkeypatch.setattr(f"{_MODULE}.BooguImageTransformer2DModel", mock_transformer_cls)
|
||||
|
||||
# Treat the dummy model id as a local path: skips hub prefetch.
|
||||
mocker.patch("os.path.exists", return_value=True)
|
||||
# Treat only the dummy model id as local. Other filesystem checks (for
|
||||
# example lazy imports in the quantization registry) must remain real.
|
||||
path_exists = os.path.exists
|
||||
mocker.patch("os.path.exists", side_effect=lambda path: str(path).startswith("dummy-boogu") or path_exists(path))
|
||||
|
||||
return {
|
||||
"inner_encoder": inner_encoder,
|
||||
"mllm_wrapper": mllm_wrapper,
|
||||
"mllm_loader": mllm_loader,
|
||||
"mllm_config_loader": mllm_config_loader,
|
||||
"mllm_builder": mllm_builder,
|
||||
"processor": mock_processor,
|
||||
"vae": mock_vae,
|
||||
"scheduler": mock_scheduler,
|
||||
@@ -128,6 +143,9 @@ def test_constructor_wires_components(boogu_pipeline, mock_dependencies):
|
||||
assert boogu_pipeline.vae_scale_factor == 8
|
||||
assert boogu_pipeline.default_sample_size == 128
|
||||
assert hasattr(boogu_pipeline, "load_weights")
|
||||
assert "quantization_config" not in mock_dependencies["mllm_loader"].call_args.kwargs
|
||||
mock_dependencies["mllm_config_loader"].assert_not_called()
|
||||
mock_dependencies["mllm_builder"].assert_not_called()
|
||||
|
||||
|
||||
def test_constructor_strips_mllm_lm_head(boogu_pipeline, mock_dependencies):
|
||||
@@ -188,7 +206,7 @@ def test_constructor_weights_sources(boogu_pipeline):
|
||||
assert source.fall_back_to_pt is True
|
||||
|
||||
|
||||
def test_constructor_routes_only_transformer_config(mock_dependencies, mocker):
|
||||
def test_constructor_rejects_unsupported_mllm_quantization(mock_dependencies, mocker):
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
|
||||
@@ -204,10 +222,143 @@ def test_constructor_routes_only_transformer_config(mock_dependencies, mocker):
|
||||
{"transformer": transformer_config, "mllm": encoder_config, "vae": None}
|
||||
),
|
||||
)
|
||||
with pytest.raises(ValueError, match="Boogu MLLM only supports FP8 quantization"):
|
||||
BooguImagePipeline(od_config=od_config)
|
||||
mock_dependencies["mllm_loader"].assert_not_called()
|
||||
mock_dependencies["mllm_config_loader"].assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("quantization_config", "quantize_mllm"),
|
||||
[
|
||||
pytest.param("fp8", True, id="global-fp8"),
|
||||
pytest.param({"mllm": {"method": "fp8"}, "transformer": {"method": "fp8"}}, True, id="mllm-and-dit-fp8"),
|
||||
pytest.param({"mllm": None, "transformer": "fp8"}, False, id="dit-only-fp8"),
|
||||
],
|
||||
)
|
||||
def test_constructor_routes_mllm_quantization(mock_dependencies, quantization_config, quantize_mllm):
|
||||
from transformers import AutoModel
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kFp8Static128BlockSym
|
||||
|
||||
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
|
||||
|
||||
od_config = OmniDiffusionConfig(
|
||||
model="dummy-boogu",
|
||||
revision="test-revision",
|
||||
tf_model_config=TransformerConfig(params={}),
|
||||
dtype=torch.bfloat16,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
pipeline = BooguImagePipeline(od_config=od_config)
|
||||
|
||||
if quantize_mllm:
|
||||
mock_dependencies["mllm_loader"].assert_not_called()
|
||||
mock_dependencies["mllm_config_loader"].assert_called_once_with(
|
||||
"dummy-boogu", subfolder="mllm", local_files_only=True, revision="test-revision"
|
||||
)
|
||||
builder = mock_dependencies["mllm_builder"]
|
||||
builder.assert_called_once()
|
||||
auto_cls, hf_config, online_quant_config = builder.call_args.args
|
||||
assert auto_cls is AutoModel
|
||||
assert hf_config is mock_dependencies["mllm_config_loader"].return_value
|
||||
assert isinstance(online_quant_config, OnlineQuantizationConfig)
|
||||
assert online_quant_config.args.linear.weight == kFp8Static128BlockSym
|
||||
assert builder.call_args.kwargs == {
|
||||
"dtype": torch.bfloat16,
|
||||
"device": pipeline._execution_device,
|
||||
"prefix": "mllm",
|
||||
"skip_modules": ("mllm.visual",),
|
||||
}
|
||||
assert pipeline.mllm is builder.return_value
|
||||
assert not pipeline.mllm.training
|
||||
assert all(not parameter.requires_grad for parameter in pipeline.mllm.parameters())
|
||||
transformer_source, mllm_source = pipeline.weights_sources
|
||||
assert transformer_source.subfolder == "transformer"
|
||||
assert mllm_source.model_or_path == "dummy-boogu"
|
||||
assert mllm_source.subfolder == "mllm"
|
||||
assert mllm_source.prefix == "mllm."
|
||||
assert mllm_source.revision == "test-revision"
|
||||
else:
|
||||
assert "quantization_config" not in mock_dependencies["mllm_loader"].call_args.kwargs
|
||||
assert pipeline.mllm is mock_dependencies["inner_encoder"]
|
||||
mock_dependencies["mllm_config_loader"].assert_not_called()
|
||||
mock_dependencies["mllm_builder"].assert_not_called()
|
||||
assert [source.subfolder for source in pipeline.weights_sources] == ["transformer"]
|
||||
assert isinstance(mock_dependencies["transformer_cls"].call_args.kwargs["quant_config"], Fp8Config)
|
||||
|
||||
|
||||
def test_constructor_preserves_mllm_ignored_layers(mock_dependencies):
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
|
||||
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
|
||||
|
||||
quant_config = Fp8Config(
|
||||
ignored_layers=["mllm.language_model.layers.0.self_attn.q_proj", "transformer.blocks.0.attn.to_q"]
|
||||
)
|
||||
od_config = OmniDiffusionConfig(
|
||||
model="dummy-boogu",
|
||||
tf_model_config=TransformerConfig(params={}),
|
||||
dtype=torch.bfloat16,
|
||||
quantization_config=quant_config,
|
||||
)
|
||||
BooguImagePipeline(od_config=od_config)
|
||||
kwargs = mock_dependencies["transformer_cls"].call_args.kwargs
|
||||
assert kwargs["quant_config"] is transformer_config
|
||||
assert kwargs["prefix"] == "transformer"
|
||||
|
||||
online_quant_config = mock_dependencies["mllm_builder"].call_args.args[2]
|
||||
assert online_quant_config.ignored_layers == quant_config.ignored_layers
|
||||
|
||||
|
||||
def test_constructor_preserves_serialized_mllm_quantization(mock_dependencies):
|
||||
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
|
||||
|
||||
mock_dependencies["mllm_config_loader"].return_value = Qwen3VLConfig(
|
||||
quantization_config={"quant_method": "fp8", "modules_to_not_convert": ["model.visual"]}
|
||||
)
|
||||
od_config = OmniDiffusionConfig(
|
||||
model="dummy-boogu-fp8",
|
||||
tf_model_config=TransformerConfig(params={}),
|
||||
dtype=torch.bfloat16,
|
||||
quantization_config="fp8",
|
||||
)
|
||||
pipeline = BooguImagePipeline(od_config=od_config)
|
||||
|
||||
# No override: HF reads the serialized checkpoint's scales and skip list.
|
||||
assert "quantization_config" not in mock_dependencies["mllm_loader"].call_args.kwargs
|
||||
assert pipeline.mllm is mock_dependencies["inner_encoder"]
|
||||
mock_dependencies["mllm_builder"].assert_not_called()
|
||||
assert [source.subfolder for source in pipeline.weights_sources] == ["transformer"]
|
||||
|
||||
|
||||
def test_load_weights_maps_mllm_and_preserves_transformer():
|
||||
from vllm_omni.diffusion.models.boogu_image.pipeline_boogu_image import BooguImagePipeline
|
||||
|
||||
pipeline = object.__new__(BooguImagePipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.mllm = nn.ModuleDict(
|
||||
{
|
||||
"language_model": nn.Linear(2, 2, bias=False),
|
||||
"visual": nn.Linear(2, 2, bias=False),
|
||||
}
|
||||
)
|
||||
pipeline.transformer = nn.Linear(2, 2, bias=False)
|
||||
language_weight = torch.full((2, 2), 1.0)
|
||||
visual_weight = torch.full((2, 2), 2.0)
|
||||
transformer_weight = torch.full((2, 2), 3.0)
|
||||
|
||||
loaded = pipeline.load_weights(
|
||||
[
|
||||
("mllm.lm_head.weight", torch.full((2, 2), 9.0)),
|
||||
("mllm.model.language_model.weight", language_weight),
|
||||
("mllm.model.visual.weight", visual_weight),
|
||||
("transformer.weight", transformer_weight),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"mllm.language_model.weight", "mllm.visual.weight", "transformer.weight"}
|
||||
torch.testing.assert_close(pipeline.mllm["language_model"].weight, language_weight)
|
||||
torch.testing.assert_close(pipeline.mllm["visual"].weight, visual_weight)
|
||||
torch.testing.assert_close(pipeline.transformer.weight, transformer_weight)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.models.ernie_image.ernie_image_transformer import _apply_rotary_emb
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_fused_rope_state():
|
||||
from vllm_omni.diffusion.models.ernie_image import fused_rope
|
||||
|
||||
fused_rope._FAILED_KEYS.clear()
|
||||
yield
|
||||
fused_rope._FAILED_KEYS.clear()
|
||||
|
||||
|
||||
def _inputs(shape: tuple[int, int, int, int]):
|
||||
batch, sequence, _, head_dim = shape
|
||||
torch.manual_seed(17)
|
||||
query = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
|
||||
key = torch.randn_like(query)
|
||||
freqs_cos = torch.randn(
|
||||
batch,
|
||||
sequence,
|
||||
head_dim // 2,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
freqs_sin = torch.randn_like(freqs_cos)
|
||||
return query, key, freqs_cos, freqs_sin
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(1, 1, 1, 128),
|
||||
(2, 257, 7, 128),
|
||||
(2, 4608, 32, 128),
|
||||
],
|
||||
)
|
||||
def test_fused_qk_rope_is_bit_exact(shape):
|
||||
from vllm_omni.diffusion.models.ernie_image import fused_rope
|
||||
|
||||
query, key, freqs_cos, freqs_sin = _inputs(shape)
|
||||
with torch.inference_mode():
|
||||
expected_query = _apply_rotary_emb(query, freqs_cos, freqs_sin)
|
||||
expected_key = _apply_rotary_emb(key, freqs_cos, freqs_sin)
|
||||
actual = fused_rope.try_fused_qk_rotary_emb(query, key, freqs_cos, freqs_sin)
|
||||
|
||||
assert actual is not None
|
||||
actual_query, actual_key = actual
|
||||
assert torch.equal(actual_query, expected_query)
|
||||
assert torch.equal(actual_key, expected_key)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
def test_launch_failure_disables_runtime_key(monkeypatch):
|
||||
from vllm_omni.diffusion.models.ernie_image import fused_rope
|
||||
|
||||
query, key, freqs_cos, freqs_sin = _inputs((1, 17, 3, 128))
|
||||
launch_count = 0
|
||||
|
||||
def failing_launch(*args):
|
||||
nonlocal launch_count
|
||||
launch_count += 1
|
||||
raise RuntimeError("injected launch failure")
|
||||
|
||||
monkeypatch.setattr(
|
||||
fused_rope,
|
||||
"_launch_fused_qk_rotary_emb",
|
||||
failing_launch,
|
||||
)
|
||||
with torch.inference_mode():
|
||||
for _ in range(2):
|
||||
assert fused_rope.try_fused_qk_rotary_emb(query, key, freqs_cos, freqs_sin) is None
|
||||
assert launch_count == 1
|
||||
assert len(fused_rope._FAILED_KEYS) == 1
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
def test_failed_runtime_key_cache_is_bounded():
|
||||
from vllm_omni.diffusion.models.ernie_image import fused_rope
|
||||
|
||||
for index in range(fused_rope._FAILED_KEYS_MAX_SIZE + 1):
|
||||
fused_rope._record_failed_runtime_key((index,))
|
||||
|
||||
assert len(fused_rope._FAILED_KEYS) == fused_rope._FAILED_KEYS_MAX_SIZE
|
||||
assert (0,) not in fused_rope._FAILED_KEYS
|
||||
assert (fused_rope._FAILED_KEYS_MAX_SIZE,) in fused_rope._FAILED_KEYS
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
def test_compile_path_does_not_launch_fused_kernel(monkeypatch):
|
||||
from vllm_omni.diffusion.models.ernie_image import fused_rope
|
||||
|
||||
query, key, freqs_cos, freqs_sin = _inputs((1, 17, 3, 128))
|
||||
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
|
||||
|
||||
def unexpected_launch(*args):
|
||||
raise AssertionError("compile path must remain native")
|
||||
|
||||
monkeypatch.setattr(
|
||||
fused_rope,
|
||||
"_launch_fused_qk_rotary_emb",
|
||||
unexpected_launch,
|
||||
)
|
||||
with torch.inference_mode():
|
||||
assert fused_rope.try_fused_qk_rotary_emb(query, key, freqs_cos, freqs_sin) is None
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
@pytest.mark.parametrize("requires_grad_input", ["query", "key", "cos", "sin"])
|
||||
def test_gradient_inputs_do_not_launch_fused_kernel(monkeypatch, requires_grad_input):
|
||||
from vllm_omni.diffusion.models.ernie_image import fused_rope
|
||||
|
||||
values = list(_inputs((1, 17, 3, 128)))
|
||||
input_index = {"query": 0, "key": 1, "cos": 2, "sin": 3}[requires_grad_input]
|
||||
values[input_index].requires_grad_(True)
|
||||
|
||||
def unexpected_launch(*args):
|
||||
raise AssertionError("autograd inputs must remain on the native path")
|
||||
|
||||
monkeypatch.setattr(
|
||||
fused_rope,
|
||||
"_launch_fused_qk_rotary_emb",
|
||||
unexpected_launch,
|
||||
)
|
||||
assert (
|
||||
fused_rope.try_fused_qk_rotary_emb(
|
||||
*values,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="NVIDIA CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
def test_mismatched_key_dtype_or_device_does_not_launch(monkeypatch):
|
||||
from vllm_omni.diffusion.models.ernie_image import fused_rope
|
||||
|
||||
query, key, freqs_cos, freqs_sin = _inputs((1, 17, 3, 128))
|
||||
|
||||
def unexpected_launch(*args):
|
||||
raise AssertionError("mismatched Q/K inputs must remain native")
|
||||
|
||||
monkeypatch.setattr(
|
||||
fused_rope,
|
||||
"_launch_fused_qk_rotary_emb",
|
||||
unexpected_launch,
|
||||
)
|
||||
for unsupported_key in (key.float(), key.cpu()):
|
||||
assert (
|
||||
fused_rope.try_fused_qk_rotary_emb(
|
||||
query,
|
||||
unsupported_key,
|
||||
freqs_cos,
|
||||
freqs_sin,
|
||||
)
|
||||
is None
|
||||
)
|
||||
@@ -1,8 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.ernie_image.ernie_image_transformer import (
|
||||
ErnieImageEmbedND3,
|
||||
_apply_qk_rotary_emb,
|
||||
_apply_rotary_emb,
|
||||
rope,
|
||||
)
|
||||
@@ -185,3 +189,30 @@ class TestErnieImageRopePositionEmbedding:
|
||||
out_diffusers = x * cos_ + rotated * sin_
|
||||
|
||||
assert torch.allclose(out_vllm, out_diffusers, atol=1e-6), "vllm-omni and diffusers outputs should match"
|
||||
|
||||
def test_apply_qk_rotary_emb_uses_native_fallback(self, monkeypatch):
|
||||
"""Unsupported routes preserve both existing eager calls exactly."""
|
||||
from vllm_omni.diffusion.models.ernie_image import (
|
||||
ernie_image_transformer,
|
||||
)
|
||||
|
||||
B, S, H, D = 2, 17, 3, 128
|
||||
query = torch.randn(B, S, H, D)
|
||||
key = torch.randn(B, S, H, D)
|
||||
freqs_cos = torch.randn(B, S, D // 2)
|
||||
freqs_sin = torch.randn(B, S, D // 2)
|
||||
monkeypatch.setattr(
|
||||
ernie_image_transformer,
|
||||
"try_fused_qk_rotary_emb",
|
||||
lambda *args: None,
|
||||
)
|
||||
|
||||
actual_query, actual_key = _apply_qk_rotary_emb(query, key, freqs_cos, freqs_sin)
|
||||
assert torch.equal(
|
||||
actual_query,
|
||||
_apply_rotary_emb(query, freqs_cos, freqs_sin),
|
||||
)
|
||||
assert torch.equal(
|
||||
actual_key,
|
||||
_apply_rotary_emb(key, freqs_cos, freqs_sin),
|
||||
)
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -362,6 +362,20 @@ def test_prepared_model_inputs_match_local_tokenization(
|
||||
local_inputs = pipeline.prepare_model_inputs(**common_kwargs)
|
||||
prepared_inputs = pipeline.prepare_model_inputs(**common_kwargs, prepared_layout=prepared_layout)
|
||||
|
||||
if with_reference_image:
|
||||
# Preparation runs without a forward context. Coverage must come from
|
||||
# the scheduler metadata copied onto this request, including every CFG row.
|
||||
reused = pipeline.prepare_model_inputs(
|
||||
**common_kwargs, prepared_layout=prepared_layout, kv_computed_tokens=(5,) * len(prefix_lens)
|
||||
)
|
||||
assert reused["cond_vae_images"] is None and reused["cond_vit_images"] is None
|
||||
partial = pipeline.prepare_model_inputs(
|
||||
**common_kwargs,
|
||||
prepared_layout=prepared_layout,
|
||||
kv_computed_tokens=(5,) * (len(prefix_lens) - 1) + (4,),
|
||||
)
|
||||
assert partial["cond_vae_images"] is not None and partial["cond_vit_images"] is not None
|
||||
|
||||
comparable_fields = (
|
||||
"input_ids",
|
||||
"position_ids",
|
||||
@@ -474,3 +488,33 @@ def test_paged_preprocess_attaches_layout_and_scheduler_kv_requests(monkeypatch)
|
||||
(12, 17, 32)
|
||||
]
|
||||
assert len(tokenizer.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("transfer_tokens, expected", [(9, [7, 5]), (4, [4, 4])])
|
||||
def test_cfg_rows_preserve_reusable_token_ids_and_native_transfer_boundary(transfer_tokens, expected):
|
||||
from vllm_omni.diffusion.diffusion_kv.kv_connector import prepare_kv_requests
|
||||
|
||||
tokenizer, image_processor = _components([12, 14])
|
||||
request = _request(guidance_scale=5.0)
|
||||
layout = _prepare(request, tokenizer, image_processor)
|
||||
layout.tokenizer_output.think_recaption_end_pos = [[7], [5]]
|
||||
layout.tokenizer_output.uncond_cfg_start_pos = [[None], [5]]
|
||||
rows = build_hunyuan_diffusion_kv_requests(request, layout)
|
||||
assert [row.prompt_token_ids for row in rows] == [
|
||||
layout.tokenizer_output.tokens[0, :7].tolist(),
|
||||
layout.tokenizer_output.tokens[1, :5].tolist(),
|
||||
]
|
||||
prepare_kv_requests(rows, {"num_transfer_tokens": transfer_tokens})
|
||||
assert [row.num_prompt_tokens for row in rows] == expected
|
||||
assert all(row.kv_transfer_params["num_transfer_tokens"] == transfer_tokens for row in rows)
|
||||
layout.tokenizer_output.uncond_cfg_start_pos = [[5]]
|
||||
with pytest.raises(ValueError, match="one boundary per CFG row"):
|
||||
build_hunyuan_diffusion_kv_requests(request, layout)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("computed, expected", [((8, 8), True), ((8, 5), False), ((0, 0), False), ((), False)])
|
||||
def test_native_image_reuse_requires_coverage_in_every_cfg_row(computed, expected):
|
||||
from vllm_omni.diffusion.models.hunyuan_image3.request_layout import native_kv_covers_cond_images
|
||||
|
||||
output = TokenizerEncodeOutput(joint_image_slices=[[slice(2, 8)], [slice(2, 6)]])
|
||||
assert native_kv_covers_cond_images(output, computed) is expected
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for HunyuanImage3 AR sampler logic (stage transitions,
|
||||
ratio restriction, comprehension blocking)."""
|
||||
|
||||
@@ -21,6 +21,10 @@ RATIO_END = 210
|
||||
RATIO_OTHER_START = 220
|
||||
RATIO_OTHER_END = 223
|
||||
TIMESTEP = 224
|
||||
CFG = 110
|
||||
IMG_TOK = 111
|
||||
JOINT_SEP = 112
|
||||
EOI = 113
|
||||
|
||||
|
||||
class FakeSamplerModel:
|
||||
@@ -47,10 +51,13 @@ class FakeSamplerModel:
|
||||
self._stage_transitions[END_OF_THINK] = [RECAPTION]
|
||||
self._stage_transitions[END_OF_RECAPTION] = [ANSWER, BOI, SIZE_TOKEN]
|
||||
|
||||
self._blocked_token_ids: set[int] = set()
|
||||
if is_comprehension:
|
||||
self._blocked_token_ids.update([BOI, SIZE_TOKEN])
|
||||
self._blocked_token_ids.update(self._all_ratio_ids)
|
||||
# Mirrors the model: the blocklist is always built so generation-mode
|
||||
# engines can apply it to per-request comprehension rows (#6088).
|
||||
self._cfg_token_id = CFG
|
||||
self._blocked_token_ids: set[int] = {BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP}
|
||||
self._blocked_token_ids.update(self._all_ratio_ids)
|
||||
self._blocked_token_ids_tensor = None
|
||||
self._sampler = None
|
||||
|
||||
# Bind the real methods from the model class.
|
||||
from vllm_omni.model_executor.models.hunyuan_image3.hunyuan_image3 import (
|
||||
@@ -59,6 +66,8 @@ class FakeSamplerModel:
|
||||
|
||||
_get_forced_token = _Real._get_forced_token
|
||||
_apply_ratio_restriction = _Real._apply_ratio_restriction
|
||||
_row_is_comprehension = staticmethod(_Real._row_is_comprehension)
|
||||
sample = _Real.sample
|
||||
|
||||
|
||||
class FakeTokenEmbedding:
|
||||
@@ -250,3 +259,114 @@ class TestForceEosAfterRatio:
|
||||
assert logits[0, EOS].item() == 0
|
||||
non_eos_max = logits[0, :EOS].max().item()
|
||||
assert non_eos_max == min_score
|
||||
|
||||
|
||||
class _RecordingSampler:
|
||||
"""Stub sampler: returns the (mutated) logits so tests can assert on them."""
|
||||
|
||||
def __call__(self, *, logits, sampling_metadata):
|
||||
return logits
|
||||
|
||||
|
||||
def _metadata(output_token_ids):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(output_token_ids=output_token_ids)
|
||||
|
||||
|
||||
class TestPerRequestTaskMode:
|
||||
"""Generation-mode engines must honor per-request comprehension rows (#6088).
|
||||
|
||||
A text chat (I2T/T2T) served by an AR+DiT deployment is marked with
|
||||
extra_args={"ar_task_mode": "comprehension"}; its rows get the stateless
|
||||
image-token blocking instead of stage-transition forcing, while untagged
|
||||
rows keep the generation behavior.
|
||||
"""
|
||||
|
||||
VOCAB = 300
|
||||
|
||||
def _run(self, model, histories, per_req_extra_args):
|
||||
logits = torch.zeros(len(histories), self.VOCAB)
|
||||
logits[:, 50] = 7.0 # ordinary text token, must survive comprehension
|
||||
model._sampler = _RecordingSampler()
|
||||
out = model.sample(logits, _metadata(histories), per_req_extra_args=per_req_extra_args)
|
||||
return out
|
||||
|
||||
def test_comprehension_row_skips_forcing_and_blocks_image_tokens(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
logits = self._run(
|
||||
model,
|
||||
histories=[[END_OF_THINK], [END_OF_THINK]],
|
||||
per_req_extra_args=[{"ar_task_mode": "comprehension"}, None],
|
||||
)
|
||||
|
||||
# Row 0 (comprehension): no </think> -> <recaption> forcing (ordinary
|
||||
# text token survives, which a force-fill would have erased) and every
|
||||
# DiT scaffold token is masked.
|
||||
assert logits[0, 50].item() == 7.0
|
||||
for tid in (BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP, RATIO_START, RATIO_OTHER_START):
|
||||
assert logits[0, tid].item() == min_score, f"token {tid} not blocked"
|
||||
|
||||
# Row 1 (generation): forced to <recaption> exactly as before.
|
||||
assert logits[1, RECAPTION].item() == 0
|
||||
assert logits[1, 50].item() == min_score
|
||||
|
||||
def test_no_extra_args_keeps_generation_behavior(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
logits = self._run(model, histories=[[END_OF_THINK]], per_req_extra_args=None)
|
||||
|
||||
assert logits[0, RECAPTION].item() == 0
|
||||
assert logits[0, 50].item() == min_score
|
||||
|
||||
def test_empty_or_foreign_extra_args_keeps_generation_behavior(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
|
||||
logits = self._run(
|
||||
model,
|
||||
histories=[[END_OF_THINK], [END_OF_THINK]],
|
||||
per_req_extra_args=[{}, {"other_key": 1}],
|
||||
)
|
||||
|
||||
assert logits[0, RECAPTION].item() == 0
|
||||
assert logits[1, RECAPTION].item() == 0
|
||||
|
||||
def test_comprehension_row_without_trigger_history(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
logits = self._run(
|
||||
model,
|
||||
histories=[[1, 2, 3]],
|
||||
per_req_extra_args=[{"ar_task_mode": "comprehension"}],
|
||||
)
|
||||
|
||||
assert logits[0, 50].item() == 7.0
|
||||
assert logits[0, CFG].item() == min_score
|
||||
|
||||
def test_static_comprehension_engine_blocks_cfg(self):
|
||||
model = FakeSamplerModel(is_comprehension=True)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
logits = self._run(model, histories=[[1, 2]], per_req_extra_args=None)
|
||||
|
||||
for tid in (BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP):
|
||||
assert logits[0, tid].item() == min_score
|
||||
assert logits[0, 50].item() == 7.0
|
||||
|
||||
def test_extra_args_shorter_than_batch_defaults_to_generation(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
|
||||
logits = self._run(
|
||||
model,
|
||||
histories=[[END_OF_THINK], [END_OF_THINK]],
|
||||
per_req_extra_args=[{"ar_task_mode": "comprehension"}],
|
||||
)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
assert logits[0, 50].item() == 7.0 # tagged row: comprehension
|
||||
assert logits[1, RECAPTION].item() == 0 # missing entry: generation
|
||||
assert logits[1, 50].item() == min_score
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
|
||||
import math
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -143,6 +144,104 @@ def test_cache_manager_registers_attention_without_adding_dense_state() -> None:
|
||||
assert mgr.state_dict() == {}
|
||||
|
||||
|
||||
def test_cache_manager_initializes_with_quantization_skip_layers(monkeypatch):
|
||||
import vllm_omni.diffusion.attention.layer as layer
|
||||
from vllm_omni.diffusion.config import set_current_diffusion_config
|
||||
from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_image3_transformer import ImageKVCacheManager
|
||||
|
||||
config = SimpleNamespace(
|
||||
diffusion_attention_config=None,
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
diffusion_kv_cache_dtype="auto",
|
||||
diffusion_kv_cache_skip_layer_indices={3},
|
||||
)
|
||||
monkeypatch.setattr(layer, "get_attn_backend_for_role", lambda **_: (layer.SDPABackend, None))
|
||||
monkeypatch.setattr(layer, "build_parallel_attention_strategy", lambda **_: object())
|
||||
with patched_mgr_env(), set_current_diffusion_config(config):
|
||||
# Keep the real Attention constructor and its skip-layer validation.
|
||||
with patch(f"{_TRANSFORMER_MODULE}.Attention", layer.Attention):
|
||||
mgr = ImageKVCacheManager(
|
||||
num_heads=NUM_HEADS,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_dim=HEAD_DIM,
|
||||
scaling=SCALING,
|
||||
prefix="model.layers.3.self_attn",
|
||||
)
|
||||
assert mgr.attn.layer_idx == 3
|
||||
assert not mgr.attn._should_apply_kv_cache_quant()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_step", [True, False])
|
||||
def test_native_connector_keeps_imported_prefix_in_paged_attention(first_step) -> None:
|
||||
from vllm_omni.diffusion.forward_context import override_paged_kv_adapter, set_forward_context
|
||||
|
||||
mgr = _make_cache_mgr()
|
||||
mgr.attn.paged_kv_active = True
|
||||
query_len, prefix_len = IMAGE_TOKEN_LEN, 19
|
||||
key, value = _make_known_kv(2 * query_len)
|
||||
spans = [[(prefix_len, prefix_len + query_len)]] * 2
|
||||
with (
|
||||
set_forward_context(),
|
||||
override_paged_kv_adapter(object()),
|
||||
patch.object(mgr, "_forward_dense_legacy", side_effect=AssertionError("dense KV path")),
|
||||
patch(f"{_TRANSFORMER_MODULE}.repeat_kv", side_effect=AssertionError("KV head expansion")),
|
||||
patch(f"{_TRANSFORMER_MODULE}.get_paged_kv_computed_tokens", return_value=(prefix_len, prefix_len)),
|
||||
):
|
||||
_call_mgr(
|
||||
mgr,
|
||||
2,
|
||||
query_len,
|
||||
prefix_len + query_len,
|
||||
key,
|
||||
value,
|
||||
first_step=first_step,
|
||||
full_attn_spans=spans,
|
||||
)
|
||||
assert len(mgr.attn.paged_calls) == 1
|
||||
assert mgr.attn.paged_calls[0][1].shape == (2, query_len, NUM_KV_HEADS, HEAD_DIM)
|
||||
assert mgr.attn.paged_metadata[0].attn_mask is None
|
||||
assert mgr.attn.paged_metadata[0].full_attn_spans == spans
|
||||
assert mgr.image_kv_cache_map is None
|
||||
assert not hasattr(mgr, "_materialize_paged_prefix")
|
||||
|
||||
|
||||
def test_legacy_tensor_prefix_still_uses_dense_cache_and_reuse() -> None:
|
||||
mgr = _make_cache_mgr()
|
||||
prefix_len, prompt_len = 3, 2
|
||||
imported = _make_known_kv(prefix_len, base=10.0)
|
||||
mgr._injected_ar_kv = [imported]
|
||||
first_len = prompt_len + IMAGE_TOKEN_LEN
|
||||
key, value = _make_known_kv(first_len, base=20.0)
|
||||
with (
|
||||
patch.object(mgr, "_forward_paged", side_effect=AssertionError("legacy entered paged path")),
|
||||
):
|
||||
_call_mgr(
|
||||
mgr,
|
||||
1,
|
||||
first_len,
|
||||
prefix_len + first_len,
|
||||
key,
|
||||
value,
|
||||
first_step=True,
|
||||
gen_timestep_scatter_index=_gen_timestep_index(1, prompt_len),
|
||||
)
|
||||
assert mgr.image_kv_cache_map is not None
|
||||
assert mgr.image_kv_cache_map[0].shape[1] == prefix_len + prompt_len
|
||||
next_key, next_value = _make_known_kv(IMAGE_TOKEN_LEN, base=30.0)
|
||||
_call_mgr(
|
||||
mgr,
|
||||
1,
|
||||
IMAGE_TOKEN_LEN,
|
||||
prefix_len + first_len,
|
||||
next_key,
|
||||
next_value,
|
||||
position_ids=torch.arange(prefix_len + prompt_len, prefix_len + first_len).unsqueeze(0),
|
||||
)
|
||||
assert len(mgr.attn.calls) == 2
|
||||
assert not mgr.attn.paged_calls
|
||||
assert mgr.attn.calls[-1][1].shape == (1, prefix_len + first_len, NUM_HEADS, HEAD_DIM)
|
||||
|
||||
|
||||
def test_scheduler_paged_kv_runs_piecewise_for_first_and_later_steps() -> None:
|
||||
from vllm_omni.diffusion.forward_context import override_paged_kv_adapter, set_forward_context
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm_omni.diffusion.models.lingbot_world.actions import (
|
||||
LINGBOT_CAMERA_ACTION_SCHEMA,
|
||||
LINGBOT_CAMERA_TRAJECTORY_SCHEMA,
|
||||
LingBotCameraControlReducer,
|
||||
camera_trajectory_from_absolute_pose,
|
||||
integrate_lingbot_camera_actions,
|
||||
parse_lingbot_camera_action_frames,
|
||||
parse_lingbot_camera_action_script,
|
||||
@@ -212,6 +213,11 @@ def test_action_integrator_matches_lingbot_motion_and_calibration() -> None:
|
||||
torch.tensor([500.0, 500.0, 208.0, 120.0]),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="resolution must be positive"):
|
||||
camera_trajectory_from_absolute_pose(trajectory.poses[:1], width=0, height=480)
|
||||
with pytest.raises(ValueError, match=r"\[frames, 4, 4\]"):
|
||||
camera_trajectory_from_absolute_pose(torch.zeros(4, 4), width=832, height=480)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data", "message"),
|
||||
|
||||
@@ -74,6 +74,17 @@ def _install_vllm_stubs() -> None:
|
||||
_SeqAllToAll4D,
|
||||
)
|
||||
|
||||
def _all_to_all_5D(value, scatter_idx=3, gather_idx=1, group=None, use_sync=False):
|
||||
# Single-rank stub: the fused q/k/v exchange is the identity when N == 1.
|
||||
del scatter_idx, gather_idx, group, use_sync
|
||||
return value
|
||||
|
||||
setattr(
|
||||
sys.modules["vllm_omni.diffusion.distributed.comm"],
|
||||
"all_to_all_5D",
|
||||
_all_to_all_5D,
|
||||
)
|
||||
|
||||
def get_sp_group():
|
||||
return SimpleNamespace(
|
||||
ulysses_world_size=1,
|
||||
@@ -588,26 +599,7 @@ def test_tp_rmsnorm_weight_loader_selects_rank_shard(monkeypatch: pytest.MonkeyP
|
||||
torch.testing.assert_close(norm.weight, torch.tensor([30.0, 40.0]))
|
||||
|
||||
|
||||
def test_single_token_text_kv_shard_owns_compact_storage(monkeypatch):
|
||||
module = _load_module()
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"get_sp_group",
|
||||
lambda: SimpleNamespace(
|
||||
ulysses_world_size=2,
|
||||
ulysses_rank=1,
|
||||
ulysses_group=None,
|
||||
),
|
||||
)
|
||||
attention = module.LingBotCrossAttention(dim=8, num_heads=4)
|
||||
full = torch.arange(8, dtype=torch.float32).reshape(1, 1, 4, 2)
|
||||
shard = attention.shard_kv_heads(full)
|
||||
torch.testing.assert_close(shard, full[:, :, 2:], rtol=0, atol=0)
|
||||
assert shard.is_contiguous()
|
||||
assert shard.untyped_storage().nbytes() == shard.numel() * shard.element_size()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attention_class", ["LingBotSelfAttention", "LingBotCrossAttention"])
|
||||
@pytest.mark.parametrize("attention_class", ["LingBotSelfAttention"])
|
||||
def test_ulysses_rejects_non_divisible_head_count(monkeypatch, attention_class):
|
||||
module = _load_module()
|
||||
monkeypatch.setattr(
|
||||
@@ -621,3 +613,35 @@ def test_ulysses_rejects_non_divisible_head_count(monkeypatch, attention_class):
|
||||
)
|
||||
with pytest.raises(ValueError, match="heads must be divisible"):
|
||||
getattr(module, attention_class)(dim=8, num_heads=4)
|
||||
|
||||
|
||||
def test_supplied_camera_modulation_matches_computing_it_inline() -> None:
|
||||
"""Passing a prebuilt modulation must be arithmetically identical to letting the block build its own."""
|
||||
module = _load_module()
|
||||
torch.manual_seed(0)
|
||||
block = module.LingBotAttentionBlock(dim=4, num_heads=2, prefix="blocks.0")
|
||||
camera = torch.randn(1, 3, 4)
|
||||
|
||||
scale, shift = block.camera_modulation(camera)
|
||||
inline_scale, inline_shift = block.camera_modulation(camera)
|
||||
|
||||
torch.testing.assert_close(scale, inline_scale, rtol=0, atol=0)
|
||||
torch.testing.assert_close(shift, inline_shift, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_the_camera_modulation_reads_only_the_camera_tokens() -> None:
|
||||
"""The reuse window is only sound because the modulation depends on nothing else that varies per forward."""
|
||||
module = _load_module()
|
||||
torch.manual_seed(0)
|
||||
block = module.LingBotAttentionBlock(dim=4, num_heads=2, prefix="blocks.0")
|
||||
camera = torch.randn(1, 3, 4)
|
||||
|
||||
first = block.camera_modulation(camera)
|
||||
# Same camera, different everything else a forward would carry: the modulation cannot move.
|
||||
second = block.camera_modulation(camera.clone())
|
||||
|
||||
torch.testing.assert_close(first[0], second[0], rtol=0, atol=0)
|
||||
torch.testing.assert_close(first[1], second[1], rtol=0, atol=0)
|
||||
# A different camera must move it, or the cache key would be hiding a real dependency.
|
||||
other = block.camera_modulation(camera + 1)
|
||||
assert not torch.equal(first[0], other[0])
|
||||
@@ -422,7 +422,17 @@ def test_build_plucker_embedding_uses_framewise_deltas_for_rotations_and_transla
|
||||
torch.testing.assert_close(result[2, 3:, 0, 1], expected_direction)
|
||||
|
||||
|
||||
def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("translation_scale", "expected_origins"),
|
||||
[
|
||||
(None, [[0.0, 0.0, 0.0], [0.3, 0.4, 0.0], [0.6, 0.8, 0.0]]),
|
||||
(0.05, [[0.0, 0.0, 0.0], [60.0, 80.0, 0.0], [120.0, 160.0, 0.0]]),
|
||||
],
|
||||
)
|
||||
def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm(
|
||||
translation_scale: float | None,
|
||||
expected_origins: list[list[float]],
|
||||
) -> None:
|
||||
poses = torch.eye(4).repeat(3, 1, 1)
|
||||
poses[:, :3, 3] = torch.tensor(
|
||||
[
|
||||
@@ -440,10 +450,37 @@ def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm()
|
||||
target_width=1,
|
||||
device=torch.device("cpu"),
|
||||
dtype=torch.float32,
|
||||
translation_scale=translation_scale,
|
||||
)
|
||||
|
||||
expected_origins = torch.tensor([[0.0, 0.0, 0.0], [0.3, 0.4, 0.0], [0.6, 0.8, 0.0]])
|
||||
torch.testing.assert_close(result[:, :3, 0, 0], expected_origins)
|
||||
torch.testing.assert_close(result[:, :3, 0, 0], torch.tensor(expected_origins))
|
||||
|
||||
from vllm_omni.diffusion.models.lingbot_world.actions import LINGBOT_CONTROLLER_TRANSLATION_UNIT
|
||||
|
||||
if translation_scale != LINGBOT_CONTROLLER_TRANSLATION_UNIT:
|
||||
return
|
||||
# Fixed controller unit keeps relative speed; max-norm would collapse both to 1.
|
||||
slow = torch.eye(4).repeat(2, 1, 1)
|
||||
slow[1, 2, 3] = 0.005
|
||||
fast = torch.eye(4).repeat(2, 1, 1)
|
||||
fast[1, 2, 3] = 0.5
|
||||
kwargs = dict(
|
||||
height=1,
|
||||
width=1,
|
||||
target_height=1,
|
||||
target_width=1,
|
||||
device=torch.device("cpu"),
|
||||
dtype=torch.float32,
|
||||
translation_scale=LINGBOT_CONTROLLER_TRANSLATION_UNIT,
|
||||
)
|
||||
slow_origin = build_plucker_embedding(_trajectory(slow), **kwargs)[1, :3, 0, 0]
|
||||
fast_origin = build_plucker_embedding(_trajectory(fast), **kwargs)[1, :3, 0, 0]
|
||||
torch.testing.assert_close(slow_origin, torch.tensor([0.0, 0.0, 0.1]))
|
||||
torch.testing.assert_close(fast_origin, torch.tensor([0.0, 0.0, 10.0]))
|
||||
wasd = torch.eye(4).repeat(2, 1, 1)
|
||||
wasd[1, 2, 3] = LINGBOT_CONTROLLER_TRANSLATION_UNIT
|
||||
wasd_origin = build_plucker_embedding(_trajectory(wasd), **kwargs)[1, :3, 0, 0]
|
||||
torch.testing.assert_close(wasd_origin, torch.tensor([0.0, 0.0, 1.0]))
|
||||
|
||||
|
||||
def test_build_plucker_embedding_interpolates_raw_poses_before_framewise_conversion() -> None:
|
||||
|
||||
@@ -837,6 +837,9 @@ def _rollout(model, mode, dtype, batch):
|
||||
frames_per_block=_FRAMES,
|
||||
max_scratch_tokens_per_branch=_FRAMES * _TOKENS_PER_FRAME,
|
||||
cross_attention_lengths={"text": 5},
|
||||
# Cross-attention keeps every local head on every SP rank, so its pool is wider than the
|
||||
# self-attention share above.
|
||||
cross_attention_kv_heads={"text": model.blocks[0].cross_attn.num_local_heads},
|
||||
device=device,
|
||||
)
|
||||
state = ARDiffusionKVState(kv, "numeric", {"main": kv.begin_request("numeric")}, num_layers=_LAYERS)
|
||||
@@ -896,7 +899,6 @@ def _worker(rank, world_size, sp_size, tp_size, mode, dtype, batch, rendezvous):
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
destroy_distributed_env,
|
||||
destroy_model_parallel,
|
||||
get_sp_group,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
@@ -948,11 +950,10 @@ def _worker(rank, world_size, sp_size, tp_size, mode, dtype, batch, rendezvous):
|
||||
bound = 1e-5 if dtype == torch.float32 else 1e-2
|
||||
error = (result - expected).double().norm() / expected.double().norm()
|
||||
assert error <= bound, f"rank={rank}, TP{tp} SP{sp}: relative L2 {error.item():.3g} > {bound}"
|
||||
local_heads = _HEADS // tp // sp
|
||||
first = (
|
||||
get_tensor_model_parallel_rank() * (_HEADS // tp)
|
||||
+ get_sp_group().ulysses_rank * local_heads
|
||||
)
|
||||
# Cross-attention K/V is replicated across the Ulysses group, not split over it, so
|
||||
# every SP rank holds its whole TP shard of heads. Only TP slices the cache now.
|
||||
local_heads = _HEADS // tp
|
||||
first = get_tensor_model_parallel_rank() * local_heads
|
||||
for (k, v), (ek, ev) in zip(cross, expected_cross, strict=True):
|
||||
tolerance = 1e-5 if dtype == torch.float32 else 2e-2
|
||||
torch.testing.assert_close(
|
||||
@@ -1033,3 +1034,56 @@ def test_temporal_rope_extension_keeps_spatial_limit():
|
||||
model._rotary_embedding(
|
||||
frames=3, height=17, width=1, start_frame=16, dtype=torch.float32, device=torch.device("cpu")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_the_camera_injector_runs_once_per_held_cache_and_gives_the_same_output() -> None:
|
||||
"""Two forwards sharing one caller-held cache: the injector runs once per block, outputs unchanged."""
|
||||
module = attention_tests._load_module()
|
||||
model = _tiny_model(module, num_layers=2).eval()
|
||||
hidden_states = torch.randn(1, 36, 1, 4, 4)
|
||||
timestep = torch.tensor([1.0])
|
||||
encoder_hidden_states = torch.randn(1, 3, 6)
|
||||
camera_hidden_states = torch.randn(1, 6 * 8 * 8, 1, 4, 4)
|
||||
|
||||
def run(cache, camera_cache=None):
|
||||
return model(
|
||||
hidden_states,
|
||||
timestep,
|
||||
encoder_hidden_states,
|
||||
camera_hidden_states,
|
||||
cache=cache,
|
||||
start_frame=0,
|
||||
update_cache=False,
|
||||
camera_modulation_cache=camera_cache,
|
||||
)
|
||||
|
||||
# Baseline: no cache, so every forward rebuilds the injector in every block.
|
||||
uncached_first = run(_cache(module, model))
|
||||
uncached_second = run(_cache(module, model))
|
||||
injector_calls = [attention_tests._record_outputs(block.cam_injector_layer1) for block in model.blocks]
|
||||
held = module.CameraModulationCache()
|
||||
cached_first = run(_cache(module, model), held)
|
||||
cached_second = run(_cache(module, model), held)
|
||||
|
||||
# One build per block for both forwards, instead of one per block per forward.
|
||||
assert [len(calls) for calls in injector_calls] == [1, 1]
|
||||
torch.testing.assert_close(cached_first, uncached_first, rtol=0, atol=0)
|
||||
torch.testing.assert_close(cached_second, uncached_second, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_timestep_projection_is_staged_into_one_static_buffer_per_shape() -> None:
|
||||
"""Two forwards' projections land in the same buffer; the blocks never see the fresh per-forward tensor."""
|
||||
module = attention_tests._load_module()
|
||||
model = _tiny_model(module, num_layers=1).eval()
|
||||
first = torch.randn(1, 8, 6, model.dim)
|
||||
second = torch.randn(1, 8, 6, model.dim)
|
||||
|
||||
staged_first = model._stage_timestep_projection(first)
|
||||
assert staged_first is not first and torch.equal(staged_first, first)
|
||||
staged_second = model._stage_timestep_projection(second)
|
||||
assert staged_second is staged_first and torch.equal(staged_second, second)
|
||||
# A different shape gets its own buffer; the first one is kept.
|
||||
other = model._stage_timestep_projection(torch.randn(1, 4, 6, model.dim))
|
||||
assert other is not staged_first and len(model._timestep_projection_buffers) == 2
|
||||
@@ -4,11 +4,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import os
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from diffusers.utils.torch_utils import randn_tensor as _diffusers_randn_tensor
|
||||
@@ -18,6 +20,7 @@ from torch import nn
|
||||
import vllm_omni.diffusion.models.lingbot_world.dmd_block as lingbot_dmd_block
|
||||
import vllm_omni.diffusion.models.lingbot_world.pipeline as lingbot_pipeline
|
||||
from tests.diffusion.models.wan2_2.conftest import noop_progress_bar
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession
|
||||
from vllm_omni.diffusion.models.interface import SupportsStepExecution, supports_step_execution
|
||||
from vllm_omni.diffusion.models.lingbot_world.actions import (
|
||||
integrate_lingbot_camera_actions,
|
||||
@@ -28,6 +31,7 @@ from vllm_omni.diffusion.models.lingbot_world.camera import (
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
from vllm_omni.experimental.ar_diffusion.capability import supports_chunk_step_grouping
|
||||
from vllm_omni.experimental.ar_diffusion.tick_protocol import (
|
||||
ARDiffusionControlInput,
|
||||
ARDiffusionTickRequest,
|
||||
@@ -126,6 +130,10 @@ class _RecordingTransformer(nn.Module):
|
||||
sink_size=3,
|
||||
)
|
||||
self.blocks = nn.ModuleList([nn.Identity(), nn.Identity()])
|
||||
for block in self.blocks:
|
||||
# The spec reads head geometry from both attentions; cross-attention keeps every local head.
|
||||
block.self_attn = SimpleNamespace(num_sp_heads=2)
|
||||
block.cross_attn = SimpleNamespace(num_local_heads=2)
|
||||
self.calls: list[dict] = []
|
||||
self.cache_allocations: list[dict] = []
|
||||
self.raise_on_call = raise_on_call
|
||||
@@ -145,6 +153,7 @@ class _RecordingTransformer(nn.Module):
|
||||
"cache_id": id(kwargs["cache"]),
|
||||
"start_frame": kwargs["start_frame"],
|
||||
"update_cache": kwargs["update_cache"],
|
||||
"camera_cache": kwargs.get("camera_modulation_cache"),
|
||||
}
|
||||
self.calls.append(call)
|
||||
if self.raise_on_call == len(self.calls):
|
||||
@@ -411,8 +420,10 @@ def _stub_pipeline_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
intrinsics=value.intrinsics[:num_frames],
|
||||
)
|
||||
|
||||
def build_plucker_embedding(value, *, height, width, target_height, target_width, device, dtype):
|
||||
del target_height, target_width
|
||||
def build_plucker_embedding(
|
||||
value, *, height, width, target_height, target_width, device, dtype, translation_scale=None
|
||||
):
|
||||
del target_height, target_width, translation_scale
|
||||
frames = value.poses.shape[0]
|
||||
data = torch.arange(frames * 6 * height * width, device=device, dtype=torch.float32)
|
||||
return data.reshape(frames, 6, height, width).to(dtype=dtype)
|
||||
@@ -531,14 +542,15 @@ def test_ar_diffusion_capability_uses_transformer_local_head_geometry() -> None:
|
||||
assert spec.sink_frames == 3
|
||||
assert [(branch.name, branch.local_index) for branch in spec.kv_branches] == [("main", 0)]
|
||||
assert spec.cross_attention_lengths == {"text": 512}
|
||||
# One condition block, committed + pending encoder caches, and decoder state.
|
||||
# One condition block, committed + pending encoder caches, the session's
|
||||
# cached prompt embedding (512 tokens x text_dim 8 x fp32), and decoder state.
|
||||
# The stub encoder retains 2 RGB frames at 16x16 and 2 feature frames at 2x2.
|
||||
encoder_bytes = (2 * 3 * 16 * 16 + 2 * 16 * 2 * 2) * 4
|
||||
assert spec.model_owned_state_bytes_per_session == 960 + 2 * encoder_bytes + 2_421_248
|
||||
assert spec.model_owned_state_bytes_per_session == 960 + 384 + 2 * encoder_bytes + 2_421_248
|
||||
|
||||
|
||||
def test_session_admission_accounts_for_the_streaming_decoder_cache() -> None:
|
||||
"""Admission includes encoder state even when the decoder cannot stream."""
|
||||
"""Admission includes encoder state and the prompt embedding even when the decoder cannot stream."""
|
||||
module = _load_pipeline_module()
|
||||
|
||||
def spec_pipeline():
|
||||
@@ -557,13 +569,29 @@ def test_session_admission_accounts_for_the_streaming_decoder_cache() -> None:
|
||||
# than the image condition alone could account for.
|
||||
assert widened - small > 2_000_000
|
||||
|
||||
# Removing decoder support leaves the condition and both encoder histories.
|
||||
# Removing decoder support leaves the condition, both encoder histories and
|
||||
# the prompt-embedding cache.
|
||||
bare = spec_pipeline()
|
||||
for attribute in ("decoder", "post_quant_conv"):
|
||||
if hasattr(bare.vae, attribute):
|
||||
delattr(bare.vae, attribute)
|
||||
bare.vae._cached_conv_counts.pop("decoder")
|
||||
assert bare.ar_diffusion_kv_cache_spec().model_owned_state_bytes_per_session == 960 + 2 * 6_656
|
||||
assert bare.ar_diffusion_kv_cache_spec().model_owned_state_bytes_per_session == 960 + 384 + 2 * 6_656
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sp_size", [1, 4])
|
||||
def test_session_admission_budgets_camera_modulation_per_rank(sp_size: int) -> None:
|
||||
pipeline = _pipeline(_load_pipeline_module())
|
||||
pipeline._ar_height, pipeline._ar_width = 480, 832
|
||||
pipeline.od_config.parallel_config.ulysses_degree = sp_size
|
||||
pipeline.transformer.config.num_layers = 40
|
||||
pipeline.transformer.config.num_attention_heads = 40
|
||||
pipeline.transformer.config.attention_head_dim = 128
|
||||
pipeline.transformer._dtype = torch.bfloat16
|
||||
forty_layers = pipeline.ar_diffusion_kv_cache_spec().model_owned_state_bytes_per_session
|
||||
pipeline.transformer.config.num_layers = 1
|
||||
one_layer = pipeline.ar_diffusion_kv_cache_spec().model_owned_state_bytes_per_session
|
||||
assert forty_layers - one_layer == 2 * 39 * (4680 // sp_size) * 5120 * 2
|
||||
|
||||
|
||||
def test_preprocess_materializes_external_inputs_before_worker_execution(tmp_path: Path) -> None:
|
||||
@@ -1330,10 +1358,34 @@ def test_first_frame_condition_and_camera_fold_match_transformer_contract() -> N
|
||||
assert expected_camera.shape == (1, 384, 3, 2, 2)
|
||||
|
||||
|
||||
def test_fixed_dmd_transition_and_cache_commit_trace() -> None:
|
||||
@pytest.mark.parametrize("value", ["true", "false", 1, 0, None])
|
||||
def test_reuse_last_step_kv_rejects_non_boolean_config(value: object) -> None:
|
||||
config = _od_config()
|
||||
config.model_config["lingbot_reuse_last_step_kv"] = value
|
||||
with pytest.raises(ValueError, match="lingbot_reuse_last_step_kv must be a bool"):
|
||||
_pipeline(_load_pipeline_module(), od_config=config)
|
||||
|
||||
|
||||
def test_reuse_last_step_kv_is_fixed_per_instance(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("VLLM_OMNI_LINGBOT_REUSE_LAST_STEP_KV", "1")
|
||||
module = _load_pipeline_module()
|
||||
config = _od_config()
|
||||
config.model_config["lingbot_reuse_last_step_kv"] = True
|
||||
reuse = _pipeline(module, od_config=config)
|
||||
default = _pipeline(module)
|
||||
config.model_config["lingbot_reuse_last_step_kv"] = False
|
||||
assert reuse._dmd_blocks.reuse_last_step_kv is True
|
||||
assert default._dmd_blocks.reuse_last_step_kv is False
|
||||
assert reuse._dmd_blocks is reuse._dmd_blocks
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reuse_last_step_kv", [False, True])
|
||||
def test_fixed_dmd_transition_and_cache_commit_trace(reuse_last_step_kv: bool) -> None:
|
||||
module = _load_pipeline_module()
|
||||
transformer = _RecordingTransformer()
|
||||
pipeline = _pipeline(module, transformer=transformer)
|
||||
config = _od_config()
|
||||
config.model_config["lingbot_reuse_last_step_kv"] = reuse_last_step_kv
|
||||
pipeline = _pipeline(module, transformer=transformer, od_config=config)
|
||||
|
||||
result = pipeline(_request())
|
||||
|
||||
@@ -1355,14 +1407,18 @@ def test_fixed_dmd_transition_and_cache_commit_trace() -> None:
|
||||
current = x0
|
||||
torch.testing.assert_close(result.output, current)
|
||||
|
||||
expected_timesteps = [timestep for timestep, _ in warped_schedule]
|
||||
if not reuse_last_step_kv:
|
||||
expected_timesteps.append(0.0)
|
||||
torch.testing.assert_close(
|
||||
torch.cat([call["timestep"] for call in transformer.calls]),
|
||||
torch.tensor([*(timestep for timestep, _ in warped_schedule), 0.0]),
|
||||
torch.cat([call["timestep"] for call in transformer.calls]), torch.tensor(expected_timesteps)
|
||||
)
|
||||
assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True]
|
||||
assert [call["start_frame"] for call in transformer.calls] == [0, 0, 0, 0, 0]
|
||||
commit_flags = [False] * (len(expected_timesteps) - 1) + [True]
|
||||
assert [call["update_cache"] for call in transformer.calls] == commit_flags
|
||||
assert [call["start_frame"] for call in transformer.calls] == [0] * len(expected_timesteps)
|
||||
assert len({call["cache_id"] for call in transformer.calls}) == 1
|
||||
torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], result.output)
|
||||
committed_latent = result.output + warped_schedule[-1][1] if reuse_last_step_kv else result.output
|
||||
torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], committed_latent)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -1669,7 +1725,6 @@ def test_typed_ticks_generate_one_global_block_and_return_standard_metadata(
|
||||
"applied_event_ids": [1],
|
||||
}
|
||||
assert pipeline._ar_sessions["world-1"].next_chunk_index == 2
|
||||
assert pipeline._ar_sessions["world-1"].prompt == "enter the snowy valley"
|
||||
assert pipeline._ar_sessions["world-1"].camera_tail is not None
|
||||
assert pipeline._ar_sessions["world-1"].camera_tail.poses.shape == (1, 4, 4)
|
||||
expected_generator = torch.Generator(device="cpu").manual_seed(17)
|
||||
@@ -1783,6 +1838,7 @@ def test_first_typed_yaw_action_uses_pre_action_identity_anchor(
|
||||
target_width=16,
|
||||
device=torch.device("cpu"),
|
||||
dtype=torch.float32,
|
||||
translation_scale=None, # None for camera_actions / action_script input (using max-norm)
|
||||
)[1:]
|
||||
|
||||
assert not torch.equal(action_embedding, neutral_embedding)
|
||||
@@ -1842,8 +1898,11 @@ def test_typed_ticks_keep_bounded_encoder_state_beyond_ten_chunks(
|
||||
pipeline(_request(sampling=sampling))
|
||||
|
||||
assert pipeline.vae.encode_inputs == []
|
||||
assert [video.shape[2] for video in pipeline.vae.encoder.inputs] == [1] + [4] * 32
|
||||
# The stub encoder's history settles after its second block; the nine
|
||||
# blocks after that reuse the settled condition without an encode.
|
||||
assert [video.shape[2] for video in pipeline.vae.encoder.inputs] == [1] + [4] * 5
|
||||
assert all(torch.count_nonzero(video) == 0 for video in pipeline.vae.encoder.inputs[1:])
|
||||
assert pipeline._ar_sessions["world-1"].condition_fixed_point is not None
|
||||
encoder_cache = pipeline._ar_sessions["world-1"].encoder_cache
|
||||
assert sum(t.numel() * t.element_size() for t in encoder_cache) == pipeline._condition_encoder_cache_bytes()
|
||||
assert pipeline.vae._enc_feat_map == ["module-owned"]
|
||||
@@ -1939,6 +1998,76 @@ def test_condition_continues_real_causal_vae_beyond_initial_horizon(
|
||||
assert pipeline._ar_sessions == {}
|
||||
|
||||
|
||||
def test_condition_encoder_stops_at_its_fixed_point_and_restarts_with_the_session(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Once a block returns the history it was given, later blocks reuse the condition without encoding.
|
||||
|
||||
The whole-clip encode is the reference for every block, so reusing the
|
||||
stored condition is only allowed to be a speedup, never a change.
|
||||
"""
|
||||
pipeline = _tiny_condition_pipeline(monkeypatch)
|
||||
pipeline._ar_diffusion_kv_state = object()
|
||||
blocks = 12
|
||||
inputs = pipeline._parse_request(_request(sampling=_SamplingParams(num_frames=(blocks * 3 - 1) * 4 + 1)))
|
||||
with torch.inference_mode():
|
||||
expected = pipeline._prepare_condition(inputs, dtype=torch.float32)
|
||||
encoder_calls = []
|
||||
original_encoder = pipeline.vae.encoder.forward
|
||||
|
||||
def encoder(*args, **kwargs):
|
||||
encoder_calls.append(len(seen))
|
||||
return original_encoder(*args, **kwargs)
|
||||
|
||||
seen: list[torch.Tensor] = []
|
||||
|
||||
def generate_block(**kwargs):
|
||||
index = len(seen) % blocks
|
||||
torch.testing.assert_close(
|
||||
kwargs["condition"], expected[:, :, index * 3 : (index + 1) * 3], rtol=1e-6, atol=1e-6
|
||||
)
|
||||
seen.append(kwargs["condition"])
|
||||
return torch.zeros_like(kwargs["condition"][:, :16])
|
||||
|
||||
monkeypatch.setattr(pipeline.vae.encoder, "forward", encoder)
|
||||
monkeypatch.setattr(pipeline, "_generate_block", generate_block)
|
||||
|
||||
def tick(index):
|
||||
pipeline(_request(sampling=_SamplingParams(extra_args=_tick_extra_args(chunk_index=index))))
|
||||
|
||||
settled_at = None
|
||||
for index in range(blocks):
|
||||
tick(index)
|
||||
state = pipeline._ar_sessions["world-1"]
|
||||
if settled_at is None and state.condition_fixed_point is not None:
|
||||
settled_at = index
|
||||
assert state.pending_encoder_cache is None and state.encoder_cache is not None
|
||||
assert settled_at is not None and 0 < settled_at < blocks - 1, settled_at
|
||||
# Three encoder passes per encoded block, none once the fixed point is stored.
|
||||
assert encoder_calls == [index for index in range(settled_at + 1) for _ in range(3)]
|
||||
# The stored condition is the session's own, not shared with the callers.
|
||||
assert all(seen[settled_at] is not later for later in seen[settled_at + 1 :])
|
||||
|
||||
# A session restarted from chunk 0 encodes again from the source image.
|
||||
pipeline.reset_ar_diffusion_session("world-1")
|
||||
tick(0)
|
||||
state = pipeline._ar_sessions["world-1"]
|
||||
assert state.condition_fixed_point is None
|
||||
assert encoder_calls[-3:] == [blocks] * 3
|
||||
|
||||
|
||||
def test_condition_encoder_fixed_point_is_released_with_the_session(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
pipeline = _tiny_condition_pipeline(monkeypatch)
|
||||
pipeline._ar_diffusion_kv_state = object()
|
||||
monkeypatch.setattr(pipeline, "_generate_block", lambda **kwargs: torch.zeros_like(kwargs["condition"][:, :16]))
|
||||
for index in range(8):
|
||||
pipeline(_request(sampling=_SamplingParams(extra_args=_tick_extra_args(chunk_index=index))))
|
||||
state = pipeline._ar_sessions["world-1"]
|
||||
assert state.condition_fixed_point is not None
|
||||
pipeline.close_ar_diffusion_session("world-1")
|
||||
assert state.condition_fixed_point is None and state.encoder_cache is None
|
||||
|
||||
|
||||
def test_condition_encoder_interleaving_retry_and_reset(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
pipeline = _tiny_condition_pipeline(monkeypatch)
|
||||
pipeline._ar_diffusion_kv_state = object()
|
||||
@@ -2015,6 +2144,7 @@ def test_stepwise_failed_chunk_releases_pending_encoder_on_close(monkeypatch: py
|
||||
transformer.raise_on_call = 1
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
|
||||
pipeline.prepare_encode(state)
|
||||
pipeline.prepare_next_chunk(state)
|
||||
session = pipeline._ar_sessions[state.request_id]
|
||||
assert session.encoder_cache is None and session.pending_encoder_cache is not None
|
||||
with pytest.raises(RuntimeError, match="forced transformer failure"):
|
||||
@@ -2037,6 +2167,7 @@ def test_stateful_condition_rejects_tiled_encoder(monkeypatch: pytest.MonkeyPatc
|
||||
state = _stepwise_state()
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
|
||||
pipeline.prepare_encode(state)
|
||||
pipeline.prepare_next_chunk(state)
|
||||
assert pipeline.vae.encoder.inputs == []
|
||||
assert all(s.encoder_cache is None and s.pending_encoder_cache is None for s in pipeline._ar_sessions.values())
|
||||
|
||||
@@ -2168,11 +2299,14 @@ def _stepwise_state(
|
||||
def _run_stepwise(pipeline, state):
|
||||
outputs = []
|
||||
pipeline.prepare_encode(state)
|
||||
pipeline.prepare_next_chunk(state)
|
||||
while not state.request_denoise_completed:
|
||||
noise = pipeline.denoise_step(None, states=[state])
|
||||
pipeline.step_scheduler(state, noise)
|
||||
if state.chunk_denoise_completed:
|
||||
outputs.append(pipeline.post_decode(state))
|
||||
if not state.request_denoise_completed:
|
||||
pipeline.prepare_next_chunk(state)
|
||||
return outputs
|
||||
|
||||
|
||||
@@ -2186,6 +2320,7 @@ def _stepwise_chunks(pipeline, state, ar_state):
|
||||
"""
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, ar_state):
|
||||
pipeline.prepare_encode(state)
|
||||
pipeline.prepare_next_chunk(state)
|
||||
while not state.request_denoise_completed:
|
||||
output = None
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, ar_state):
|
||||
@@ -2193,10 +2328,46 @@ def _stepwise_chunks(pipeline, state, ar_state):
|
||||
pipeline.step_scheduler(state, noise)
|
||||
if state.chunk_denoise_completed:
|
||||
output = pipeline.post_decode(state)
|
||||
if not state.request_denoise_completed:
|
||||
pipeline.prepare_next_chunk(state)
|
||||
if output is not None:
|
||||
yield output
|
||||
|
||||
|
||||
def test_stepwise_block_shares_one_camera_cache_across_its_five_forwards() -> None:
|
||||
"""The four probes and the commit of one stepwise block reuse one camera cache; the next block gets its own.
|
||||
|
||||
Request mode creates one cache per loop; the stepwise path's forwards are
|
||||
separate scheduler steps, so the pipeline holds the block's cache and
|
||||
passes it to each one.
|
||||
"""
|
||||
module = _load_pipeline_module()
|
||||
transformer = _RecordingTransformer()
|
||||
pipeline = _pipeline(module, transformer=transformer)
|
||||
pipeline._ar_height = 16
|
||||
pipeline._ar_width = 16
|
||||
state = _stepwise_state(num_frames=21)
|
||||
state.sampling.output_type = "latent"
|
||||
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
|
||||
outputs = _run_stepwise(pipeline, state)
|
||||
|
||||
assert len(outputs) == 2 and len(transformer.calls) == 10
|
||||
caches = [call["camera_cache"] for call in transformer.calls]
|
||||
assert all(c is not None for c in caches)
|
||||
assert len({id(c) for c in caches[:5]}) == 1 and len({id(c) for c in caches[5:]}) == 1
|
||||
assert caches[0] is not caches[5]
|
||||
# The block's cache does not outlive its commit.
|
||||
assert "camera_cache" not in state.extra
|
||||
|
||||
|
||||
def test_pipeline_declares_chunk_step_grouping() -> None:
|
||||
"""A realtime block's probes and commit may run without a scheduler cycle between them."""
|
||||
module = _load_pipeline_module()
|
||||
assert module.LingBotWorldCausalDMDPipeline.supports_chunk_step_grouping is True
|
||||
assert supports_chunk_step_grouping(_pipeline(module))
|
||||
|
||||
|
||||
def test_pipeline_declares_step_execution_support() -> None:
|
||||
module = _load_pipeline_module()
|
||||
pipeline = _pipeline(module)
|
||||
@@ -2246,10 +2417,13 @@ def test_forward_rejects_a_stepwise_camera_action_script(extra_args) -> None:
|
||||
pipeline(_request(sampling=sampling))
|
||||
|
||||
|
||||
def test_stepwise_progress_metadata_and_commit_trace() -> None:
|
||||
@pytest.mark.parametrize("reuse_last_step_kv", [False, True])
|
||||
def test_stepwise_progress_metadata_and_commit_trace(reuse_last_step_kv: bool) -> None:
|
||||
module = _load_pipeline_module()
|
||||
transformer = _RecordingTransformer()
|
||||
pipeline = _pipeline(module, transformer=transformer)
|
||||
config = _od_config()
|
||||
config.model_config["lingbot_reuse_last_step_kv"] = reuse_last_step_kv
|
||||
pipeline = _pipeline(module, transformer=transformer, od_config=config)
|
||||
pipeline._ar_height = 16
|
||||
pipeline._ar_width = 16
|
||||
state = _stepwise_state(num_frames=21)
|
||||
@@ -2263,8 +2437,9 @@ def test_stepwise_progress_metadata_and_commit_trace() -> None:
|
||||
assert [output.chunk_index for output in outputs] == [0, 1]
|
||||
assert all(output.total_chunks == 2 for output in outputs)
|
||||
assert outputs[-1].finished is True
|
||||
assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True] * 2
|
||||
assert [call["start_frame"] for call in transformer.calls] == [0] * 5 + [3] * 5
|
||||
calls_per_chunk = 4 if reuse_last_step_kv else 5
|
||||
assert [call["update_cache"] for call in transformer.calls] == ([False] * (calls_per_chunk - 1) + [True]) * 2
|
||||
assert [call["start_frame"] for call in transformer.calls] == [0] * calls_per_chunk + [3] * calls_per_chunk
|
||||
assert fake.commits == ["main", "main"]
|
||||
for chunk_index, output in enumerate(outputs):
|
||||
metadata = output.output["metadata"]["ar_diffusion"]
|
||||
@@ -2325,7 +2500,10 @@ def test_stepwise_matches_tick_transformer_trace_and_latents(num_chunks) -> None
|
||||
torch.testing.assert_close(step_call["camera_hidden_states"], tick_call["camera_hidden_states"])
|
||||
assert stepwise_state.extra["condition"].shape[2] == 3
|
||||
assert pipeline.vae.encode_inputs == []
|
||||
assert [video.shape[2] for video in pipeline.vae.encoder.inputs] == ([1] + [4] * (num_chunks * 3 - 1)) * 2
|
||||
# The stub encoder's history settles after its second block, so both
|
||||
# modes encode two blocks and reuse the settled condition after that.
|
||||
encoded_blocks = min(num_chunks, 2)
|
||||
assert [video.shape[2] for video in pipeline.vae.encoder.inputs] == ([1] + [4] * (encoded_blocks * 3 - 1)) * 2
|
||||
assert stepwise_fake.commits == ["main"] * num_chunks
|
||||
assert len(stepwise_outputs) == num_chunks and stepwise_outputs[-1].finished
|
||||
assert pipeline.vae.decode_inputs == []
|
||||
@@ -2394,21 +2572,16 @@ def test_stepwise_emits_latents_without_touching_the_vae() -> None:
|
||||
assert pipeline.vae.decode_inputs == []
|
||||
|
||||
|
||||
def _capture_video_processor(monkeypatch) -> list[tuple[tuple[int, ...], str]]:
|
||||
"""Record what reaches postprocess_video, and name each chunk's output."""
|
||||
import diffusers.video_processor as video_processor_module
|
||||
def _capture_pixel_conversion(monkeypatch) -> list[tuple[int, ...]]:
|
||||
"""Record each decoded block reaching the uint8 conversion, and name its output."""
|
||||
module = _load_pipeline_module()
|
||||
processed: list[tuple[int, ...]] = []
|
||||
|
||||
processed: list[tuple[tuple[int, ...], str]] = []
|
||||
def uint8_frames(video):
|
||||
processed.append(tuple(video.shape))
|
||||
return f"frames-{len(processed)}"
|
||||
|
||||
class VideoProcessor:
|
||||
def __init__(self, *, vae_scale_factor):
|
||||
assert vae_scale_factor == 8
|
||||
|
||||
def postprocess_video(self, video, *, output_type):
|
||||
processed.append((tuple(video.shape), output_type))
|
||||
return f"frames-{len(processed)}"
|
||||
|
||||
monkeypatch.setattr(video_processor_module, "VideoProcessor", VideoProcessor)
|
||||
monkeypatch.setattr(module, "_uint8_frames", uint8_frames)
|
||||
return processed
|
||||
|
||||
|
||||
@@ -2422,7 +2595,7 @@ def _streaming_pipeline(module):
|
||||
def test_stepwise_decodes_each_chunk_for_streaming_consumers(monkeypatch) -> None:
|
||||
"""A non-latent request must stream pixels under the primary "video" key."""
|
||||
module = _load_pipeline_module()
|
||||
processed = _capture_video_processor(monkeypatch)
|
||||
processed = _capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
state = _stepwise_state(num_frames=21)
|
||||
state.sampling.output_type = "np"
|
||||
@@ -2431,7 +2604,7 @@ def test_stepwise_decodes_each_chunk_for_streaming_consumers(monkeypatch) -> Non
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, fake):
|
||||
outputs = _run_stepwise(pipeline, state)
|
||||
|
||||
assert [output_type for _, output_type in processed] == ["np", "np"]
|
||||
assert processed == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
|
||||
assert [output.output["payload"] for output in outputs] == [{"video": "frames-1"}, {"video": "frames-2"}]
|
||||
# Identity metadata survives the switch to pixel output.
|
||||
assert [output.output["metadata"]["ar_diffusion"]["chunk_index"] for output in outputs] == [0, 1]
|
||||
@@ -2445,7 +2618,7 @@ def test_stepwise_chunks_continue_one_sessions_temporal_decode(monkeypatch) -> N
|
||||
the same latents would: 9 frames, then 12.
|
||||
"""
|
||||
module = _load_pipeline_module()
|
||||
processed = _capture_video_processor(monkeypatch)
|
||||
processed = _capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
state = _stepwise_state(num_frames=21)
|
||||
state.sampling.output_type = "np"
|
||||
@@ -2454,7 +2627,7 @@ def test_stepwise_chunks_continue_one_sessions_temporal_decode(monkeypatch) -> N
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, fake):
|
||||
_run_stepwise(pipeline, state)
|
||||
|
||||
assert [shape for shape, _ in processed] == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
|
||||
assert processed == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
|
||||
# One opening frame for the session, not one per chunk.
|
||||
assert pipeline.vae.decoder.first_chunk_flags == [True] + [False] * 5
|
||||
# The whole-clip decode path is not involved, and the module-owned cache
|
||||
@@ -2463,10 +2636,68 @@ def test_stepwise_chunks_continue_one_sessions_temporal_decode(monkeypatch) -> N
|
||||
assert pipeline.vae._feat_map == ["module-owned"]
|
||||
|
||||
|
||||
def test_prepare_next_chunk_rejects_live_camera_with_scripted_paths() -> None:
|
||||
"""Scripted/cached camera must not silently ignore mid-generation camera events."""
|
||||
module = _load_pipeline_module()
|
||||
pipeline = _pipeline(module, transformer=_RecordingTransformer())
|
||||
pipeline._ar_height = 16
|
||||
pipeline._ar_width = 16
|
||||
|
||||
# Scenario 1, input pre-scripted camera trajectory via action_script
|
||||
state = _stepwise_state(num_frames=21)
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
|
||||
pipeline.prepare_encode(state)
|
||||
assert state.extra.get("camera_action_script") is not None
|
||||
state.interaction_sessions["camera"] = CameraSession(has_received_input=True)
|
||||
with pytest.raises(ValueError, match="camera_action_script"):
|
||||
pipeline.prepare_next_chunk(state)
|
||||
|
||||
# Scenario 2, input pre-scripted camera trajectory via embedding cache
|
||||
poses = torch.eye(4).repeat(21, 1, 1)
|
||||
trajectory = _CameraTrajectory(
|
||||
poses=poses,
|
||||
intrinsics=torch.tensor([[100.0, 100.0, 8.0, 8.0]]).repeat(21, 1),
|
||||
)
|
||||
state = _stepwise_state(num_frames=21)
|
||||
state.sampling.extra_args["_lingbot_camera_trajectory"] = trajectory
|
||||
state.sampling.extra_args.pop("_lingbot_camera_action_script")
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)): # pyright: ignore[reportArgumentType]
|
||||
pipeline.prepare_encode(state)
|
||||
assert state.extra.get("camera_embedding_cache") is not None
|
||||
state.interaction_sessions["camera"] = CameraSession(has_received_input=True)
|
||||
with pytest.raises(ValueError, match="camera_embedding_cache"):
|
||||
pipeline.prepare_next_chunk(state)
|
||||
|
||||
|
||||
def test_peek_chunk_media_matches_streaming_decoder_frame_counts(monkeypatch) -> None:
|
||||
"""Camera timelines must track the streaming decoder's 9-then-12 media counts."""
|
||||
module = _load_pipeline_module()
|
||||
_capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
state = _stepwise_state(num_frames=21)
|
||||
state.sampling.output_type = "np"
|
||||
state.sampling.fps = 16.0
|
||||
fake = _FakeARState(state.request_id)
|
||||
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, fake):
|
||||
pipeline.prepare_encode(state)
|
||||
first = pipeline.peek_chunk_media(state)
|
||||
assert (first.num_media_frames, first.fps, first.num_latent_frames) == (9, 16.0, 3)
|
||||
|
||||
pipeline.prepare_next_chunk(state)
|
||||
while not state.chunk_denoise_completed:
|
||||
noise = pipeline.denoise_step(None, states=[state])
|
||||
pipeline.step_scheduler(state, noise)
|
||||
pipeline.post_decode(state)
|
||||
|
||||
second = pipeline.peek_chunk_media(state)
|
||||
assert (second.num_media_frames, second.fps, second.num_latent_frames) == (12, 16.0, 3)
|
||||
|
||||
|
||||
def test_streaming_decode_state_is_owned_by_the_session(monkeypatch) -> None:
|
||||
"""Decoder state is keyed by request id and released with the AR session."""
|
||||
module = _load_pipeline_module()
|
||||
_capture_video_processor(monkeypatch)
|
||||
_capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
state = _stepwise_state(request_id="req-stream", num_frames=21)
|
||||
state.sampling.output_type = "np"
|
||||
@@ -2490,7 +2721,7 @@ def test_streaming_decode_state_is_owned_by_the_session(monkeypatch) -> None:
|
||||
def test_streaming_decode_state_is_dropped_on_session_reset(monkeypatch) -> None:
|
||||
"""A reset session must not resume the temporal context it just abandoned."""
|
||||
module = _load_pipeline_module()
|
||||
_capture_video_processor(monkeypatch)
|
||||
_capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
state = _stepwise_state(request_id="req-reset", num_frames=21)
|
||||
state.sampling.output_type = "np"
|
||||
@@ -2514,7 +2745,7 @@ def test_streaming_decode_keeps_interleaved_sessions_isolated(monkeypatch) -> No
|
||||
interleaving the scheduler actually produces.
|
||||
"""
|
||||
module = _load_pipeline_module()
|
||||
processed = _capture_video_processor(monkeypatch)
|
||||
processed = _capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
states = []
|
||||
for request_id in ("req-a", "req-b"):
|
||||
@@ -2547,7 +2778,7 @@ def test_streaming_decode_keeps_interleaved_sessions_isolated(monkeypatch) -> No
|
||||
assert pipeline.vae.decoder.first_chunk_flags == (
|
||||
[True, False, False] + [True, False, False] + [False] * 3 + [False] * 3
|
||||
)
|
||||
assert [shape for shape, _ in processed] == [
|
||||
assert processed == [
|
||||
(1, 3, 9, 16, 16),
|
||||
(1, 3, 9, 16, 16),
|
||||
(1, 3, 12, 16, 16),
|
||||
@@ -2558,7 +2789,7 @@ def test_streaming_decode_keeps_interleaved_sessions_isolated(monkeypatch) -> No
|
||||
def test_streaming_decode_failure_drops_the_half_advanced_cache(monkeypatch) -> None:
|
||||
"""A failed chunk must not leave a cache a later chunk would continue from."""
|
||||
module = _load_pipeline_module()
|
||||
_capture_video_processor(monkeypatch)
|
||||
_capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
state = _stepwise_state(request_id="req-boom", num_frames=21)
|
||||
state.sampling.output_type = "np"
|
||||
@@ -2592,13 +2823,14 @@ def test_streaming_decode_receives_rescaled_latents(monkeypatch) -> None:
|
||||
never reaches, so this asserts on what ``post_quant_conv`` received.
|
||||
"""
|
||||
module = _load_pipeline_module()
|
||||
_capture_video_processor(monkeypatch)
|
||||
_capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
state = _stepwise_state(num_frames=21)
|
||||
state.sampling.output_type = "np"
|
||||
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
|
||||
pipeline.prepare_encode(state)
|
||||
pipeline.prepare_next_chunk(state)
|
||||
while not state.chunk_denoise_completed:
|
||||
pipeline.step_scheduler(state, pipeline.denoise_step(None, states=[state]))
|
||||
model_space = state.latents.clone()
|
||||
@@ -2624,7 +2856,7 @@ def test_streaming_decode_is_reported_to_the_profiler(monkeypatch) -> None:
|
||||
perf regression in the streaming decoder would be too.
|
||||
"""
|
||||
module = _load_pipeline_module()
|
||||
_capture_video_processor(monkeypatch)
|
||||
_capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _pipeline(
|
||||
module,
|
||||
transformer=_RecordingTransformer(),
|
||||
@@ -2652,21 +2884,18 @@ def test_streaming_decode_is_reported_to_the_profiler(monkeypatch) -> None:
|
||||
def test_streaming_decode_falls_back_when_the_vae_would_tile(monkeypatch) -> None:
|
||||
"""Tiled decode drives its own cache, so a tiling shape keeps per-chunk decode."""
|
||||
module = _load_pipeline_module()
|
||||
processed = _capture_video_processor(monkeypatch)
|
||||
processed = _capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
# A latent of 2x2 exceeds a one-latent-cell tile, so vae.decode would tile.
|
||||
_enable_vae_tiling(pipeline, tile_sample_min=8)
|
||||
outputs = [
|
||||
pipeline._decode_chunk_to_pixels(torch.zeros(1, 16, 3, 2, 2), output_type="np", session_id="tiled")
|
||||
for _ in range(2)
|
||||
]
|
||||
outputs = [pipeline._decode_chunk_to_pixels(torch.zeros(1, 16, 3, 2, 2), session_id="tiled") for _ in range(2)]
|
||||
|
||||
# One whole-clip decode per AR block, each seeing only that block's frames.
|
||||
assert [tuple(latents.shape) for latents in pipeline.vae.decode_inputs] == [(1, 16, 3, 2, 2)] * 2
|
||||
assert pipeline.vae.decoder.first_chunk_flags == []
|
||||
assert pipeline._streaming_decode_states == {}
|
||||
# Nine frames per chunk instead of 9 then 12: the frame loss being removed.
|
||||
assert [shape for shape, _ in processed] == [(1, 3, 9, 16, 16), (1, 3, 9, 16, 16)]
|
||||
assert processed == [(1, 3, 9, 16, 16), (1, 3, 9, 16, 16)]
|
||||
assert outputs == ["frames-1", "frames-2"]
|
||||
|
||||
|
||||
@@ -2678,7 +2907,7 @@ def test_streaming_decode_survives_tiling_enabled_below_the_tile_threshold(monke
|
||||
on a configuration that would never have tiled.
|
||||
"""
|
||||
module = _load_pipeline_module()
|
||||
processed = _capture_video_processor(monkeypatch)
|
||||
processed = _capture_pixel_conversion(monkeypatch)
|
||||
pipeline = _streaming_pipeline(module)
|
||||
# A 2x2 latent is within a tile this size, so vae.decode would not tile.
|
||||
_enable_vae_tiling(pipeline, tile_sample_min=256)
|
||||
@@ -2690,7 +2919,7 @@ def test_streaming_decode_survives_tiling_enabled_below_the_tile_threshold(monke
|
||||
|
||||
assert pipeline.vae.decode_inputs == []
|
||||
assert pipeline.vae.decoder.first_chunk_flags == [True] + [False] * 5
|
||||
assert [shape for shape, _ in processed] == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
|
||||
assert processed == [(1, 3, 9, 16, 16), (1, 3, 12, 16, 16)]
|
||||
|
||||
|
||||
def test_registry_and_model_exports_resolve_official_pipeline_class_name() -> None:
|
||||
@@ -2833,3 +3062,193 @@ def test_stepwise_requests_with_the_same_prompt_share_one_encode() -> None:
|
||||
|
||||
assert encoded == ["move through the room"]
|
||||
assert states[0].prompt_embeds is states[1].prompt_embeds
|
||||
|
||||
|
||||
def test_session_text_caches_publish_key_only_after_a_successful_build() -> None:
|
||||
module = _load_pipeline_module()
|
||||
pipeline = _pipeline(module)
|
||||
state = module._LingBotARSessionState()
|
||||
invalidations: list[bool] = []
|
||||
fail_next = {"raise": False}
|
||||
|
||||
def fake_text_caches(prompt_embeds, *, invalidate):
|
||||
invalidations.append(invalidate)
|
||||
if fail_next["raise"]:
|
||||
fail_next["raise"] = False
|
||||
raise RuntimeError("injected failure inside _ar_text_caches")
|
||||
return ["caches"]
|
||||
|
||||
pipeline._ar_text_caches = fake_text_caches
|
||||
embeds = torch.ones(1, 512, 8)
|
||||
caches = lambda prompt, length=512, e=embeds: pipeline._session_text_caches( # noqa: E731
|
||||
e, prompt=prompt, max_sequence_length=length, session_state=state
|
||||
)
|
||||
|
||||
# First build of a session: runner state is empty -> no invalidation, key published.
|
||||
caches("A")
|
||||
assert invalidations == [False] and state.text_cache_key == ("A", 512, str(embeds.dtype))
|
||||
assert state.text_cache_dirty is False
|
||||
# Same inputs: reuse without invalidation.
|
||||
caches("A")
|
||||
assert invalidations == [False, False]
|
||||
# Regression (Codex review): B's build fails after invalidating -> no key is
|
||||
# left behind, so retrying A must invalidate again instead of consuming
|
||||
# partial/mixed K/V.
|
||||
fail_next["raise"] = True
|
||||
with pytest.raises(RuntimeError, match="injected failure"):
|
||||
caches("B")
|
||||
assert state.text_cache_key is None and state.text_cache_dirty is True
|
||||
caches("A")
|
||||
assert invalidations[-2:] == [True, True] and state.text_cache_key == ("A", 512, str(embeds.dtype))
|
||||
assert state.text_cache_dirty is False
|
||||
# The key covers every embedding-defining input: same text, other length or dtype -> rebuild.
|
||||
caches("A", length=256)
|
||||
caches("A", e=embeds.to(torch.bfloat16))
|
||||
assert invalidations[-2:] == [True, True]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# a source image is decoded once while the file behind it is unchanged
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_png(path, colour=(10, 20, 30)):
|
||||
Image.new("RGB", (64, 64), colour).save(path)
|
||||
return path
|
||||
|
||||
|
||||
def _counting_decode(module):
|
||||
"""Replace the file decode with a counter, so the tests measure decodes rather than pixels."""
|
||||
decoded: list[str] = []
|
||||
original = module._decode_source_image_file
|
||||
|
||||
def decode(path):
|
||||
decoded.append(os.fspath(path))
|
||||
return Image.new("RGB", (64, 64), (1, 2, 3))
|
||||
|
||||
module._decode_source_image_file = decode
|
||||
return decoded, original
|
||||
|
||||
|
||||
def test_the_same_source_image_is_decoded_once(tmp_path: Path) -> None:
|
||||
module = _load_pipeline_module()
|
||||
module._decode_source_image_fingerprinted.cache_clear()
|
||||
path = _write_png(tmp_path / "frame.png")
|
||||
decoded, original = _counting_decode(module)
|
||||
try:
|
||||
first = module._load_source_image(path)
|
||||
second = module._load_source_image(path)
|
||||
assert decoded == [os.fspath(os.path.realpath(path))] or len(decoded) == 1
|
||||
# Every caller owns its copy: mutating one must not reach the next caller or the cache.
|
||||
assert first is not second
|
||||
first.putpixel((0, 0), (255, 255, 255))
|
||||
assert module._load_source_image(path).getpixel((0, 0)) == (1, 2, 3)
|
||||
assert len(decoded) == 1
|
||||
finally:
|
||||
module._decode_source_image_file = original
|
||||
module._decode_source_image_fingerprinted.cache_clear()
|
||||
|
||||
|
||||
def test_a_rewritten_source_image_is_decoded_again(tmp_path: Path) -> None:
|
||||
module = _load_pipeline_module()
|
||||
module._decode_source_image_fingerprinted.cache_clear()
|
||||
path = _write_png(tmp_path / "frame.png")
|
||||
decoded, original = _counting_decode(module)
|
||||
try:
|
||||
module._load_source_image(path)
|
||||
module._load_source_image(path)
|
||||
assert len(decoded) == 1
|
||||
# Same path, different bytes: the mtime/size in the key must force a fresh decode.
|
||||
os.utime(path, ns=(0, 0))
|
||||
_write_png(path, colour=(200, 100, 50))
|
||||
module._load_source_image(path)
|
||||
assert len(decoded) == 2
|
||||
finally:
|
||||
module._decode_source_image_file = original
|
||||
module._decode_source_image_fingerprinted.cache_clear()
|
||||
|
||||
|
||||
def test_an_image_rewritten_during_the_decode_is_never_cached(tmp_path: Path) -> None:
|
||||
module = _load_pipeline_module()
|
||||
module._decode_source_image_fingerprinted.cache_clear()
|
||||
path = _write_png(tmp_path / "frame.png")
|
||||
decoded: list[str] = []
|
||||
original = module._decode_source_image_file
|
||||
|
||||
def decode_then_rewrite(p):
|
||||
decoded.append(os.fspath(p))
|
||||
_write_png(path, colour=(len(decoded), 0, 0))
|
||||
os.utime(path, ns=(len(decoded) * 1_000_000, len(decoded) * 1_000_000))
|
||||
return Image.new("RGB", (64, 64), (1, 2, 3))
|
||||
|
||||
module._decode_source_image_file = decode_then_rewrite
|
||||
try:
|
||||
module._load_source_image(path)
|
||||
# The entry is keyed by the fingerprint taken before the decode, which the rewrite invalidated: the
|
||||
# next call must decode again rather than serve the image that was decoded mid-rewrite.
|
||||
module._load_source_image(path)
|
||||
assert len(decoded) == 2
|
||||
finally:
|
||||
module._decode_source_image_file = original
|
||||
module._decode_source_image_fingerprinted.cache_clear()
|
||||
|
||||
|
||||
def test_an_unstattable_path_keeps_upstream_validation(tmp_path: Path) -> None:
|
||||
module = _load_pipeline_module()
|
||||
module._decode_source_image_fingerprinted.cache_clear()
|
||||
missing = tmp_path / "does-not-exist.png"
|
||||
# The uncached path owns the error contract; caching must not change which exception a caller sees.
|
||||
with pytest.raises(ValueError):
|
||||
module._load_source_image(missing)
|
||||
assert module._decode_source_image_fingerprinted.cache_info().currsize == 0
|
||||
|
||||
|
||||
def test_the_text_cache_is_sized_by_cross_attention_heads_not_the_self_attention_share() -> None:
|
||||
"""Cross-attention replicates heads per rank, so its pool must not inherit the self-attention head count."""
|
||||
module = _load_pipeline_module()
|
||||
pipeline = _pipeline(module)
|
||||
pipeline.transformer.blocks[0].self_attn = SimpleNamespace(num_sp_heads=1)
|
||||
pipeline.transformer.blocks[0].cross_attn = SimpleNamespace(num_local_heads=4)
|
||||
|
||||
spec = pipeline.ar_diffusion_kv_cache_spec()
|
||||
|
||||
assert spec.num_kv_heads == 1
|
||||
assert spec.cross_attention_kv_heads == {"text": 4}
|
||||
assert spec.cross_attention_lengths == {"text": 512}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_type", ["np", "pil"])
|
||||
def test_streamed_chunk_is_uint8_frames_whatever_pixel_output_type_was_asked(monkeypatch, output_type) -> None:
|
||||
"""A realtime tick delivers (F, H, W, 3) uint8 for every pixel output type; only latent differs."""
|
||||
module = _load_pipeline_module()
|
||||
pipeline = _streaming_pipeline(module)
|
||||
state = _stepwise_state(num_frames=21)
|
||||
state.sampling.output_type = output_type
|
||||
|
||||
with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
|
||||
outputs = _run_stepwise(pipeline, state)
|
||||
|
||||
videos = [output.output["payload"]["video"] for output in outputs]
|
||||
assert [type(video) for video in videos] == [np.ndarray, np.ndarray]
|
||||
assert [video.dtype for video in videos] == [np.uint8, np.uint8]
|
||||
assert [video.shape for video in videos] == [(9, 16, 16, 3), (12, 16, 16, 3)]
|
||||
|
||||
|
||||
def test_uint8_frames_match_the_pil_conversion_byte_for_byte() -> None:
|
||||
"""The device-side conversion is the PIL path's arithmetic, so the bytes are the same."""
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
module = _load_pipeline_module()
|
||||
generator = torch.Generator().manual_seed(7)
|
||||
# Cover the clamp on both sides and values that sit exactly on rounding edges.
|
||||
video = torch.rand(1, 3, 5, 8, 8, generator=generator) * 2.4 - 1.2
|
||||
video[0, :, 0, 0, :4] = torch.tensor([-1.0, 1.0, 0.0, 1 / 255 - 1.0])
|
||||
|
||||
reference = VideoProcessor(vae_scale_factor=8).postprocess_video(video, output_type="pil")[0]
|
||||
expected = np.stack([np.asarray(frame) for frame in reference])
|
||||
|
||||
frames = module._uint8_frames(video)
|
||||
|
||||
assert frames.dtype == np.uint8 and frames.shape == (5, 8, 8, 3)
|
||||
assert frames.flags["C_CONTIGUOUS"]
|
||||
np.testing.assert_array_equal(frames, expected)
|
||||
@@ -0,0 +1,239 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from diffusers.models.embeddings import apply_rotary_emb
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_qk_rope_signature_caches():
|
||||
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
|
||||
|
||||
longcat._FAILED_QK_ROPE_SIGNATURES.clear()
|
||||
yield
|
||||
longcat._FAILED_QK_ROPE_SIGNATURES.clear()
|
||||
|
||||
|
||||
def _inputs(sequence: int = 5, head_dim: int = 8):
|
||||
q = torch.randn(2, sequence, 3, head_dim, dtype=torch.bfloat16)
|
||||
k = torch.randn_like(q)
|
||||
cos = torch.randn(sequence, head_dim)
|
||||
sin = torch.randn_like(cos)
|
||||
return q, k, (cos, sin)
|
||||
|
||||
|
||||
def _reference(q, k, rotary_emb):
|
||||
return (
|
||||
apply_rotary_emb(q, rotary_emb, sequence_dim=1),
|
||||
apply_rotary_emb(k, rotary_emb, sequence_dim=1),
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_rotary_emb_resolves_threshold_once_only_when_enabled(monkeypatch):
|
||||
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
|
||||
|
||||
calls = 0
|
||||
|
||||
def resolve(default):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
assert default == 512
|
||||
return 37
|
||||
|
||||
monkeypatch.setattr(longcat, "fused_qk_norm_rope_min_tokens", resolve)
|
||||
txt_cos, txt_sin = torch.randn(2, 8), torch.randn(2, 8)
|
||||
img_cos, img_sin = torch.randn(3, 8), torch.randn(3, 8)
|
||||
native = longcat._prepare_rotary_emb(txt_cos, txt_sin, img_cos, img_sin, enable_fusion=False)
|
||||
fused = longcat._prepare_rotary_emb(txt_cos, txt_sin, img_cos, img_sin, enable_fusion=True)
|
||||
|
||||
assert calls == 1
|
||||
assert len(native) == 2
|
||||
assert len(fused) == 3 and fused[2] == 37
|
||||
assert torch.equal(native[0], fused[0])
|
||||
assert torch.equal(native[1], fused[1])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "sp_size"),
|
||||
[
|
||||
("unsupported", 1),
|
||||
("compile", 1),
|
||||
("grad", 1),
|
||||
("capture", 1),
|
||||
("sequence_parallel", 2),
|
||||
("below_threshold", 1),
|
||||
],
|
||||
)
|
||||
def test_qk_rope_non_eager_modes_use_exact_original_path(monkeypatch, mode, sp_size):
|
||||
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
|
||||
|
||||
torch.manual_seed(1)
|
||||
q, k, rotary_emb = _inputs()
|
||||
expected = _reference(q, k, rotary_emb)
|
||||
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: mode == "compile")
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: mode == "capture")
|
||||
monkeypatch.setattr(longcat, "fused_qk_rope_supported", lambda *args: mode != "unsupported")
|
||||
monkeypatch.setattr(
|
||||
longcat,
|
||||
"fused_qk_rope",
|
||||
lambda *args: (_ for _ in ()).throw(AssertionError("fallback called fused op")),
|
||||
)
|
||||
|
||||
context = nullcontext() if mode == "grad" else torch.no_grad()
|
||||
min_tokens = q.shape[0] * q.shape[1] + 1 if mode == "below_threshold" else 0
|
||||
with context:
|
||||
actual = longcat._apply_qk_rope(q, k, rotary_emb, sp_size, min_tokens)
|
||||
|
||||
assert torch.equal(actual[0], expected[0])
|
||||
assert torch.equal(actual[1], expected[1])
|
||||
|
||||
|
||||
def test_qk_rope_eligible_path_skips_runtime_parity_check(monkeypatch):
|
||||
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
|
||||
|
||||
torch.manual_seed(2)
|
||||
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: False)
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
|
||||
monkeypatch.setattr(longcat, "fused_qk_rope_supported", lambda *args: True)
|
||||
fused_calls = 0
|
||||
|
||||
def fused(query, key, cos, sin):
|
||||
nonlocal fused_calls
|
||||
fused_calls += 1
|
||||
return query + 1, key + 2
|
||||
|
||||
monkeypatch.setattr(
|
||||
longcat,
|
||||
"_apply_qk_rope_reference",
|
||||
lambda *args: (_ for _ in ()).throw(AssertionError("eligible path ran the native parity check")),
|
||||
)
|
||||
monkeypatch.setattr(longcat, "fused_qk_rope", fused)
|
||||
|
||||
with torch.no_grad():
|
||||
q, k, rotary_emb = _inputs()
|
||||
actual = longcat._apply_qk_rope(q, k, rotary_emb, 1, 0)
|
||||
|
||||
assert fused_calls == 1
|
||||
assert torch.equal(actual[0], q + 1)
|
||||
assert torch.equal(actual[1], k + 2)
|
||||
|
||||
|
||||
def test_qk_rope_launch_failure_permanently_falls_back(monkeypatch):
|
||||
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
|
||||
|
||||
torch.manual_seed(3)
|
||||
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: False)
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
|
||||
monkeypatch.setattr(longcat, "fused_qk_rope_supported", lambda *args: True)
|
||||
fused_calls = 0
|
||||
|
||||
def broken_fused(query, key, cos, sin):
|
||||
nonlocal fused_calls
|
||||
fused_calls += 1
|
||||
raise RuntimeError("kernel failed")
|
||||
|
||||
monkeypatch.setattr(longcat, "fused_qk_rope", broken_fused)
|
||||
q, k, rotary_emb = _inputs()
|
||||
expected = _reference(q, k, rotary_emb)
|
||||
with torch.no_grad():
|
||||
first = longcat._apply_qk_rope(q, k, rotary_emb, 1, 0)
|
||||
second = longcat._apply_qk_rope(q, k, rotary_emb, 1, 0)
|
||||
|
||||
assert fused_calls == 1
|
||||
assert len(longcat._FAILED_QK_ROPE_SIGNATURES) == 1
|
||||
for actual in (first, second):
|
||||
assert torch.equal(actual[0], expected[0])
|
||||
assert torch.equal(actual[1], expected[1])
|
||||
|
||||
|
||||
def test_failed_qk_rope_signature_cache_is_bounded():
|
||||
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
|
||||
|
||||
for index in range(longcat._FAILED_QK_ROPE_SIGNATURES_MAX_SIZE + 1):
|
||||
longcat._record_failed_qk_rope_signature((index,))
|
||||
|
||||
assert len(longcat._FAILED_QK_ROPE_SIGNATURES) == longcat._FAILED_QK_ROPE_SIGNATURES_MAX_SIZE
|
||||
assert (0,) not in longcat._FAILED_QK_ROPE_SIGNATURES
|
||||
assert (longcat._FAILED_QK_ROPE_SIGNATURES_MAX_SIZE,) in longcat._FAILED_QK_ROPE_SIGNATURES
|
||||
|
||||
|
||||
class _FakeQKV(torch.nn.Module):
|
||||
def __init__(self, heads: int, head_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.num_heads = heads
|
||||
self.num_kv_heads = heads
|
||||
self.width = heads * head_dim
|
||||
|
||||
def forward(self, x):
|
||||
return torch.cat((x, x + 1, x + 2), dim=-1), None
|
||||
|
||||
|
||||
class _AddNorm(torch.nn.Module):
|
||||
def __init__(self, value: float) -> None:
|
||||
super().__init__()
|
||||
self.value = value
|
||||
|
||||
def forward(self, x):
|
||||
return x + self.value
|
||||
|
||||
|
||||
class _CaptureAttention(torch.nn.Module):
|
||||
def forward(self, query, key, value, metadata=None):
|
||||
del key, value, metadata
|
||||
return query
|
||||
|
||||
|
||||
class _PassthroughLinear(torch.nn.Module):
|
||||
def forward(self, x):
|
||||
return x, None
|
||||
|
||||
|
||||
def test_dual_attention_keeps_native_norm_and_text_image_concat_order(monkeypatch):
|
||||
from vllm_omni.diffusion.models.longcat_image import longcat_image_transformer as longcat
|
||||
|
||||
attention = longcat.LongCatImageAttention.__new__(longcat.LongCatImageAttention)
|
||||
torch.nn.Module.__init__(attention)
|
||||
attention.parallel_config = SimpleNamespace(sequence_parallel_size=1)
|
||||
attention.head_dim = 4
|
||||
attention.added_kv_proj_dim = 8
|
||||
attention.to_qkv = _FakeQKV(2, 4)
|
||||
attention.add_kv_proj = _FakeQKV(2, 4)
|
||||
attention.norm_q = _AddNorm(10)
|
||||
attention.norm_k = _AddNorm(20)
|
||||
attention.norm_added_q = _AddNorm(100)
|
||||
attention.norm_added_k = _AddNorm(200)
|
||||
attention.attn = _CaptureAttention()
|
||||
attention.to_out = _PassthroughLinear()
|
||||
attention.to_add_out = _PassthroughLinear()
|
||||
|
||||
captured = []
|
||||
|
||||
def paired(query, key, rotary_emb, sp_size, min_tokens):
|
||||
captured.append((query.clone(), key.clone(), rotary_emb, sp_size, min_tokens))
|
||||
return query, key
|
||||
|
||||
monkeypatch.setattr(longcat, "_apply_qk_rope", paired)
|
||||
image = torch.randn(1, 3, 8)
|
||||
text = torch.randn(1, 2, 8)
|
||||
rotary_pair = (torch.randn(5, 4), torch.randn(5, 4))
|
||||
rotary_emb = (*rotary_pair, 0)
|
||||
image_out, text_out = attention(image, text, rotary_emb)
|
||||
|
||||
image_q = image.unflatten(-1, (2, 4))
|
||||
text_q = text.unflatten(-1, (2, 4))
|
||||
expected_q = torch.cat((text_q + 100, image_q + 10), dim=1)
|
||||
expected_k = torch.cat((text_q + 1 + 200, image_q + 1 + 20), dim=1)
|
||||
assert len(captured) == 1
|
||||
assert torch.equal(captured[0][0], expected_q)
|
||||
assert torch.equal(captured[0][1], expected_k)
|
||||
assert captured[0][2][0] is rotary_pair[0]
|
||||
assert captured[0][2][1] is rotary_pair[1]
|
||||
assert captured[0][3:] == (1, 0)
|
||||
assert image_out.shape == image.shape
|
||||
assert text_out.shape == text.shape
|
||||
@@ -14,12 +14,13 @@ from tests.diffusion.models.magi2.test_native_packing import (
|
||||
)
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from vllm_omni.diffusion.models.magi2.sampler_magi2 import CFGConfig
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.core_model,
|
||||
*hardware_marks(res={"cuda": "L4"}, num_cards=1),
|
||||
pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA"),
|
||||
pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="requires CUDA"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm_omni.diffusion.offloader.distributed_layerwise_backend import (
|
||||
DistributedLayerwiseOffloadBackend,
|
||||
DistributedLayerwiseOffloadHook,
|
||||
)
|
||||
from vllm_omni.diffusion.offloader.plan_resolver import ResolvedComponent
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -46,12 +47,6 @@ class _WeightSourceStub:
|
||||
prefix: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PipelineModulesStub:
|
||||
dits: list[nn.Module]
|
||||
dit_names: list[str]
|
||||
|
||||
|
||||
_WORLD_SIZE = 4
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model]
|
||||
|
||||
@@ -189,7 +184,7 @@ def _load_mmap_transform_and_reconstruct(
|
||||
backend._mmap_transforms_by_tensor_id = {}
|
||||
backend._load_weights_via_mmap(
|
||||
pipeline,
|
||||
_PipelineModulesStub(dits=[target], dit_names=["transformer"]),
|
||||
(ResolvedComponent(path="transformer", module=target, selected=True),),
|
||||
plan_result.plan,
|
||||
)
|
||||
|
||||
|
||||
@@ -90,6 +90,21 @@ def test_hsdp_policy_shards_individual_preview_layers():
|
||||
assert not condition("pre_adapter", model.pre_adapter)
|
||||
|
||||
|
||||
def test_layerwise_offload_contract() -> None:
|
||||
from vllm_omni.diffusion.offloader import get_blocks_from_dit
|
||||
|
||||
model = Magi2PreviewTransformer(_tiny_config(num_layers=2))
|
||||
|
||||
assert model._layerwise_offload_blocks_attrs == ["block"]
|
||||
assert "block" in dict(model.named_children())
|
||||
assert list(model.block) == list(model.block.layers)
|
||||
assert model.layers is model.block.layers
|
||||
|
||||
attr_names, blocks = get_blocks_from_dit(model)
|
||||
assert attr_names == ["block"]
|
||||
assert blocks == list(model.block.layers)
|
||||
|
||||
|
||||
def test_data_proxy_keeps_output_layout_request_scoped() -> None:
|
||||
proxy = Magi2DataProxy(Magi2PreviewDataProxyConfig(time_channel_dim=0))
|
||||
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig, TransformerConfig
|
||||
from vllm_omni.diffusion.models.mammoth_moda2 import pipeline_mammothmoda2_dit
|
||||
from vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit import (
|
||||
MammothModa2DiTPipeline,
|
||||
_build_mammoth_config,
|
||||
_root_weight_source,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _raw_config() -> dict:
|
||||
return {
|
||||
"model_type": "mammothmoda2",
|
||||
"llm_config": {
|
||||
"model_type": "mammothmoda2_qwen2_5_vl",
|
||||
"text_config": {
|
||||
"model_type": "mammothmoda2_qwen2_5_vl_text",
|
||||
"hidden_size": 8,
|
||||
"gen_vocab_start_index": 100,
|
||||
},
|
||||
},
|
||||
"gen_vae_config": {"block_out_channels": [8, 8]},
|
||||
"gen_dit_config": {"hidden_size": 8, "in_channels": 4},
|
||||
}
|
||||
|
||||
|
||||
def _od_config() -> OmniDiffusionConfig:
|
||||
return OmniDiffusionConfig(
|
||||
model="/models/MammothModa2-Preview",
|
||||
model_class_name="MammothModa2DiTPipeline",
|
||||
tf_model_config=TransformerConfig.from_dict(_raw_config()),
|
||||
)
|
||||
|
||||
|
||||
def test_build_mammoth_config_uses_shared_transformer_projection() -> None:
|
||||
config = _build_mammoth_config(_od_config())
|
||||
assert config.model_type == "mammothmoda2"
|
||||
assert config.get_text_config().hidden_size == 8
|
||||
assert config.gen_dit_config == {"hidden_size": 8, "in_channels": 4}
|
||||
|
||||
|
||||
def test_build_mammoth_config_rejects_empty_shared_projection() -> None:
|
||||
config = _od_config()
|
||||
config.tf_model_config = TransformerConfig()
|
||||
with pytest.raises(ValueError, match="root checkpoint config"):
|
||||
_build_mammoth_config(config)
|
||||
|
||||
|
||||
def test_root_weight_source_loads_combined_checkpoint_once() -> None:
|
||||
source = _root_weight_source(_od_config())
|
||||
assert source.model_or_path == "/models/MammothModa2-Preview"
|
||||
assert source.subfolder is None
|
||||
assert source.prefix == ""
|
||||
assert source.fall_back_to_pt is True
|
||||
|
||||
|
||||
def test_root_weight_source_forwards_revision() -> None:
|
||||
config = _od_config()
|
||||
config.revision = "rev-7"
|
||||
assert _root_weight_source(config).revision == "rev-7"
|
||||
|
||||
|
||||
def test_pipeline_declares_native_components_and_single_request_mode_only() -> None:
|
||||
assert MammothModa2DiTPipeline._dit_modules == ["gen_transformer"]
|
||||
assert MammothModa2DiTPipeline._encoder_modules == ["gen_image_condition_refiner"]
|
||||
assert MammothModa2DiTPipeline._vae_modules == ["gen_vae"]
|
||||
assert MammothModa2DiTPipeline.supports_request_batch is False
|
||||
assert MammothModa2DiTPipeline.supports_step_execution is False
|
||||
|
||||
|
||||
def test_mammoth_postprocess_denormalizes_nonnegative_raw_vae_output() -> None:
|
||||
factory = getattr(pipeline_mammothmoda2_dit, "get_mammoth_moda2_post_process_func", None)
|
||||
assert factory is not None
|
||||
|
||||
images = factory(_od_config())(torch.zeros(1, 3, 2, 2))
|
||||
|
||||
assert len(images) == 1
|
||||
assert images[0].getpixel((0, 0)) == (128, 128, 128)
|
||||
|
||||
|
||||
def test_mammoth_postprocess_is_registered() -> None:
|
||||
from vllm_omni.diffusion.registry import _DIFFUSION_POST_PROCESS_FUNCS
|
||||
|
||||
assert _DIFFUSION_POST_PROCESS_FUNCS["MammothModa2DiTPipeline"] == "get_mammoth_moda2_post_process_func"
|
||||
|
||||
|
||||
def test_root_weight_source_rejects_missing_model_path() -> None:
|
||||
config = _od_config()
|
||||
config.model = None
|
||||
with pytest.raises(ValueError, match="model path"):
|
||||
_root_weight_source(config)
|
||||
|
||||
|
||||
def _pipeline_shell() -> MammothModa2DiTPipeline:
|
||||
pipeline = object.__new__(MammothModa2DiTPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.config = _build_mammoth_config(_od_config())
|
||||
pipeline._llm_hidden_size = 8
|
||||
return pipeline
|
||||
|
||||
|
||||
def _batch(
|
||||
*,
|
||||
request_id: str = "req-7",
|
||||
prompt: object | None = None,
|
||||
sampling: OmniDiffusionSamplingParams | None = None,
|
||||
) -> DiffusionRequestBatch:
|
||||
if prompt is None:
|
||||
prompt = {
|
||||
"prompt": "",
|
||||
"height": 32,
|
||||
"width": 48,
|
||||
"additional_information": {
|
||||
"full_hidden_states": torch.arange(32, dtype=torch.float32).reshape(4, 8),
|
||||
"full_token_ids": [10, 11, 100, 101],
|
||||
"answer_start_index": 2,
|
||||
},
|
||||
}
|
||||
if sampling is None:
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=32,
|
||||
width=48,
|
||||
seed=42,
|
||||
guidance_scale=4.0,
|
||||
num_inference_steps=7,
|
||||
extra_args={"cfg_range": [0.2, 0.8]},
|
||||
)
|
||||
return DiffusionRequestBatch([OmniDiffusionRequest(prompt=prompt, sampling_params=sampling, request_id=request_id)])
|
||||
|
||||
|
||||
def test_parse_request_uses_standard_sampling_fields() -> None:
|
||||
parsed = _pipeline_shell()._parse_request(_batch())
|
||||
assert parsed.request_id == "req-7"
|
||||
assert (parsed.height, parsed.width) == (32, 48)
|
||||
assert parsed.num_inference_steps == 7
|
||||
assert parsed.text_guidance_scale == 4.0
|
||||
assert parsed.cfg_range == (0.2, 0.8)
|
||||
assert parsed.seed == 42
|
||||
assert parsed.answer_start_index == 2
|
||||
|
||||
|
||||
def test_parse_request_prefers_legacy_sampling_overrides() -> None:
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
guidance_scale=3.0,
|
||||
num_inference_steps=5,
|
||||
seed=9,
|
||||
extra_args={"text_guidance_scale": 6.0, "num_inference_steps": 11, "cfg_range": [0.0, 0.5]},
|
||||
)
|
||||
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
|
||||
assert parsed.text_guidance_scale == 6.0
|
||||
assert parsed.num_inference_steps == 11
|
||||
assert parsed.cfg_range == (0.0, 0.5)
|
||||
|
||||
|
||||
def test_parse_request_falls_back_to_request_level_sampling_values() -> None:
|
||||
prompt = _batch().prompts[0]
|
||||
prompt["additional_information"].update(
|
||||
text_guidance_scale=[1.5],
|
||||
num_inference_steps=[3],
|
||||
cfg_range=[0.25, 0.75],
|
||||
)
|
||||
|
||||
parsed = _pipeline_shell()._parse_request(_batch(prompt=prompt, sampling=OmniDiffusionSamplingParams()))
|
||||
|
||||
assert parsed.text_guidance_scale == 1.5
|
||||
assert parsed.num_inference_steps == 3
|
||||
assert parsed.cfg_range == (0.25, 0.75)
|
||||
|
||||
|
||||
def test_parse_request_standard_fields_precede_request_level_fallbacks() -> None:
|
||||
prompt = _batch().prompts[0]
|
||||
prompt["additional_information"].update(
|
||||
text_guidance_scale=[1.5],
|
||||
num_inference_steps=[3],
|
||||
)
|
||||
sampling = OmniDiffusionSamplingParams(guidance_scale=4.0, num_inference_steps=7)
|
||||
|
||||
parsed = _pipeline_shell()._parse_request(_batch(prompt=prompt, sampling=sampling))
|
||||
|
||||
assert parsed.text_guidance_scale == 4.0
|
||||
assert parsed.num_inference_steps == 7
|
||||
|
||||
|
||||
def test_parse_request_rejects_multiple_requests_and_outputs() -> None:
|
||||
pipeline = _pipeline_shell()
|
||||
batch = _batch()
|
||||
batch.requests.append(_batch(request_id="req-8").requests[0])
|
||||
with pytest.raises(ValueError, match="exactly one request"):
|
||||
pipeline._parse_request(batch)
|
||||
with pytest.raises(ValueError, match="num_outputs_per_prompt=1"):
|
||||
pipeline._parse_request(_batch(sampling=OmniDiffusionSamplingParams(num_outputs_per_prompt=2)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cfg_range", "message"),
|
||||
[
|
||||
([0.5], "two values"),
|
||||
([-0.1, 0.5], "0 <= start <= end <= 1"),
|
||||
([0.7, 0.2], "0 <= start <= end <= 1"),
|
||||
([0.2, 1.1], "0 <= start <= end <= 1"),
|
||||
],
|
||||
)
|
||||
def test_parse_request_rejects_invalid_cfg_range(cfg_range, message) -> None:
|
||||
batch = _batch(sampling=OmniDiffusionSamplingParams(extra_args={"cfg_range": cfg_range}))
|
||||
with pytest.raises(ValueError, match=message):
|
||||
_pipeline_shell()._parse_request(batch)
|
||||
|
||||
|
||||
def test_parse_request_requires_real_ar_conditions() -> None:
|
||||
with pytest.raises(ValueError, match="req-missing"):
|
||||
_pipeline_shell()._parse_request(_batch(request_id="req-missing", prompt={"prompt": "draw a cat"}))
|
||||
|
||||
|
||||
def test_parse_request_rejects_hidden_state_token_count_mismatch() -> None:
|
||||
batch = _batch()
|
||||
batch.prompts[0]["additional_information"]["full_hidden_states"] = torch.zeros(3, 8)
|
||||
with pytest.raises(ValueError, match="hidden-state/token-count mismatch"):
|
||||
_pipeline_shell()._parse_request(batch)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("height", "width", "message"),
|
||||
[
|
||||
(0, 32, "Invalid image size"),
|
||||
(32, -1, "Invalid image size"),
|
||||
(30, 32, "multiples of 16"),
|
||||
(32, 31, "multiples of 16"),
|
||||
],
|
||||
)
|
||||
def test_parse_request_rejects_invalid_dimensions(height, width, message) -> None:
|
||||
batch = _batch()
|
||||
batch.prompts[0].update(height=height, width=width)
|
||||
with pytest.raises(ValueError, match=message):
|
||||
_pipeline_shell()._parse_request(batch)
|
||||
|
||||
|
||||
def test_parse_request_rejects_explicit_zero_steps() -> None:
|
||||
with pytest.raises(ValueError, match="num_inference_steps must be positive"):
|
||||
_pipeline_shell()._parse_request(_batch(sampling=OmniDiffusionSamplingParams(num_inference_steps=0)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("standard_steps", "expected_steps"), [(7, 7), (None, 50)])
|
||||
def test_parse_request_falls_through_null_legacy_step_count(standard_steps, expected_steps) -> None:
|
||||
sampling = OmniDiffusionSamplingParams(num_inference_steps=standard_steps, extra_args={"num_inference_steps": None})
|
||||
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
|
||||
assert parsed.num_inference_steps == expected_steps
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("standard_guidance", "expected_guidance"), [(4.0, 4.0), (None, 9.0)])
|
||||
def test_parse_request_falls_through_null_legacy_guidance(standard_guidance, expected_guidance) -> None:
|
||||
sampling = OmniDiffusionSamplingParams(guidance_scale=standard_guidance, extra_args={"text_guidance_scale": None})
|
||||
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
|
||||
assert parsed.text_guidance_scale == expected_guidance
|
||||
|
||||
|
||||
def test_parse_request_defaults_null_cfg_range() -> None:
|
||||
sampling = OmniDiffusionSamplingParams(extra_args={"cfg_range": None})
|
||||
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
|
||||
assert parsed.cfg_range == (0.0, 1.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("generators", [[], [torch.Generator(), torch.Generator()]])
|
||||
def test_parse_request_rejects_invalid_generator_list_cardinality(generators) -> None:
|
||||
sampling = OmniDiffusionSamplingParams(generator=generators)
|
||||
with pytest.raises(ValueError, match="exactly one generator.*req-generators"):
|
||||
_pipeline_shell()._parse_request(_batch(request_id="req-generators", sampling=sampling))
|
||||
|
||||
|
||||
def test_parse_request_synthesizes_dummy_ar_conditions() -> None:
|
||||
batch = _batch(
|
||||
request_id="dummy_req_id",
|
||||
prompt={"prompt": "dummy run"},
|
||||
sampling=OmniDiffusionSamplingParams(height=512, width=512, seed=1, guidance_scale=0.0, num_inference_steps=2),
|
||||
)
|
||||
parsed = _pipeline_shell()._parse_request(batch)
|
||||
assert parsed.full_hidden_states.shape == (2, 8)
|
||||
assert parsed.full_token_ids == [0, 100]
|
||||
assert parsed.answer_start_index == 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeTransformerConfig:
|
||||
in_channels: int = 4
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeTimeCaptionEmbed:
|
||||
image_embedder: nn.Module | None = None
|
||||
|
||||
|
||||
class _FakeTransformer(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.anchor = nn.Parameter(torch.zeros(1))
|
||||
self.config = _FakeTransformerConfig()
|
||||
self.time_caption_embed = _FakeTimeCaptionEmbed()
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, *, hidden_states, **kwargs):
|
||||
self.calls += 1
|
||||
return torch.zeros_like(hidden_states)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeVaeConfig:
|
||||
scaling_factor: float | None = None
|
||||
shift_factor: float | None = None
|
||||
|
||||
|
||||
class _FakeVae(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.anchor = nn.Parameter(torch.zeros(1))
|
||||
self.config = _FakeVaeConfig()
|
||||
|
||||
def decode(self, latents, return_dict=False):
|
||||
assert return_dict is False
|
||||
return (torch.zeros(1, 3, 32, 48, dtype=latents.dtype),)
|
||||
|
||||
|
||||
class _FakeScheduler:
|
||||
def __init__(self) -> None:
|
||||
self.timesteps = torch.tensor([])
|
||||
self.requested_steps = None
|
||||
|
||||
def set_timesteps(self, *, num_inference_steps, device, num_tokens):
|
||||
self.requested_steps = num_inference_steps
|
||||
self.timesteps = torch.arange(num_inference_steps, device=device, dtype=torch.float32)
|
||||
|
||||
def step(self, model_pred, timestep, latents, return_dict=False):
|
||||
assert return_dict is False
|
||||
return (latents - model_pred,)
|
||||
|
||||
|
||||
def test_forward_returns_diffusion_output_with_request_sampling(mocker) -> None:
|
||||
pipeline = _pipeline_shell()
|
||||
pipeline.gen_transformer = _FakeTransformer()
|
||||
pipeline.gen_image_condition_refiner = None
|
||||
pipeline.gen_vae = _FakeVae()
|
||||
pipeline.gen_freqs_cis = torch.zeros(1)
|
||||
scheduler = _FakeScheduler()
|
||||
captured = {}
|
||||
|
||||
def fake_randn_tensor(shape, *, generator, device, dtype):
|
||||
captured["shape"] = shape
|
||||
captured["seed"] = generator.initial_seed()
|
||||
return torch.zeros(shape, device=device, dtype=dtype)
|
||||
|
||||
module = "vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit"
|
||||
mocker.patch(f"{module}.FlowMatchEulerDiscreteScheduler", return_value=scheduler)
|
||||
mocker.patch(f"{module}.randn_tensor", side_effect=fake_randn_tensor)
|
||||
result = pipeline.forward(
|
||||
_batch(
|
||||
sampling=OmniDiffusionSamplingParams(
|
||||
height=32, width=48, seed=42, guidance_scale=1.0, num_inference_steps=2
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(result, DiffusionOutput)
|
||||
assert result.output.shape == (1, 3, 32, 48)
|
||||
assert captured["shape"] == (1, 4, 4, 6)
|
||||
assert captured["seed"] == 42
|
||||
assert scheduler.requested_steps == 2
|
||||
assert pipeline.gen_transformer.calls == 2
|
||||
|
||||
|
||||
def test_forward_rejects_missing_visual_tokens_before_model_access() -> None:
|
||||
prompt = {
|
||||
"prompt": "",
|
||||
"additional_information": {
|
||||
"full_hidden_states": torch.zeros(3, 8),
|
||||
"full_token_ids": [10, 11, 12],
|
||||
"answer_start_index": 2,
|
||||
},
|
||||
}
|
||||
with pytest.raises(ValueError, match="no visual-token hidden states.*req-empty"):
|
||||
_pipeline_shell().forward(_batch(request_id="req-empty", prompt=prompt))
|
||||
@@ -0,0 +1,679 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, DiffusionParallelConfig, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.models.minimax_h3.fasth3_checkpoint import (
|
||||
FASTH3_BASE_MODEL,
|
||||
FASTH3_V2_BASE_SCHEDULE,
|
||||
FASTH3_V2_MODEL_ID,
|
||||
FastH3CheckpointSpec,
|
||||
)
|
||||
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
|
||||
MiniMaxH3DiTArchConfig,
|
||||
MiniMaxH3DiTModel,
|
||||
)
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
resolve_minimax_h3_diffusion_model_path,
|
||||
)
|
||||
from vllm_omni.diffusion.sched.sigma_schedule import DMD2SigmaSchedule
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.model_executor.models.minimax_h3.checkpoint import (
|
||||
is_minimax_h3_modular,
|
||||
resolve_minimax_h3_model_root,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _fastvideo_metadata() -> dict[str, object]:
|
||||
"""Return the release-owned metadata read from fastvideo_inference.json."""
|
||||
return {
|
||||
"model_id": FASTH3_V2_MODEL_ID,
|
||||
"schema_version": "fasth3-inference-contract-v1",
|
||||
"dmd_denoising_steps": [999, 874, 749, 624, 500, 375, 250, 125],
|
||||
"transformer_forwards": 8,
|
||||
"video_scheduler_shift": 10.0,
|
||||
"audio_scheduler_shift": 3.0,
|
||||
"guidance_scale": 1.0,
|
||||
"attention_backend": "VIDEO_SPARSE_ATTN_H3",
|
||||
"vsa_sparsity": 0.8,
|
||||
"vsa_tile_size": 64,
|
||||
"task": "t2av",
|
||||
}
|
||||
|
||||
|
||||
def _sampling(
|
||||
*,
|
||||
num_inference_steps: int | str | None = 8,
|
||||
guidance_scale: float | None = 1.0,
|
||||
extra_args: dict[str, object] | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
timesteps: torch.Tensor | None = None,
|
||||
sigmas: list[float] | None = None,
|
||||
) -> OmniDiffusionSamplingParams:
|
||||
return OmniDiffusionSamplingParams(
|
||||
# Keep the string form to exercise malformed client input validation.
|
||||
num_inference_steps=num_inference_steps, # type: ignore[arg-type]
|
||||
guidance_scale=guidance_scale,
|
||||
extra_args=extra_args or {},
|
||||
lora_request=lora_request,
|
||||
timesteps=timesteps,
|
||||
sigmas=sigmas,
|
||||
)
|
||||
|
||||
|
||||
def _od_config(
|
||||
*,
|
||||
backend: str = "FASTVIDEO_VSA",
|
||||
topk: int | None = None,
|
||||
per_role: dict[str, AttentionSpec] | None = None,
|
||||
lora_path: str | None = None,
|
||||
ring_degree: int = 1,
|
||||
allgather_degree: int = 1,
|
||||
) -> OmniDiffusionConfig:
|
||||
attention_config = AttentionConfig(
|
||||
default=AttentionSpec(backend=backend, fastvideo_vsa_topk=topk),
|
||||
per_role=per_role or {},
|
||||
)
|
||||
return OmniDiffusionConfig(
|
||||
diffusion_attention_config=attention_config,
|
||||
lora_path=lora_path,
|
||||
parallel_config=DiffusionParallelConfig(ring_degree=ring_degree, allgather_degree=allgather_degree),
|
||||
)
|
||||
|
||||
|
||||
def test_v2_release_has_nine_sigma_nodes_and_eight_intervals():
|
||||
assert FASTH3_V2_BASE_SCHEDULE.base_schedule == (
|
||||
0.999,
|
||||
0.874,
|
||||
0.749,
|
||||
0.624,
|
||||
0.5,
|
||||
0.375,
|
||||
0.25,
|
||||
0.125,
|
||||
0.0,
|
||||
)
|
||||
assert len(FASTH3_V2_BASE_SCHEDULE.base_schedule) == 9
|
||||
assert FASTH3_V2_BASE_SCHEDULE.num_inference_steps == 8
|
||||
assert FASTH3_V2_BASE_SCHEDULE.shifted_sigmas(10.0) == pytest.approx(
|
||||
[
|
||||
0.9998999099189271,
|
||||
0.985788405143244,
|
||||
0.9675752486758817,
|
||||
0.94316807738815,
|
||||
0.9090909090909091,
|
||||
0.8571428571428571,
|
||||
0.7692307692307693,
|
||||
0.5882352941176471,
|
||||
0.0,
|
||||
],
|
||||
abs=1e-12,
|
||||
)
|
||||
assert FASTH3_V2_BASE_SCHEDULE.shifted_sigmas(3.0) == pytest.approx(
|
||||
[
|
||||
0.9996664442961973,
|
||||
0.9541484716157204,
|
||||
0.8995196156925539,
|
||||
0.8327402135231315,
|
||||
0.75,
|
||||
0.6428571428571429,
|
||||
0.5,
|
||||
0.3,
|
||||
0.0,
|
||||
],
|
||||
abs=1e-12,
|
||||
)
|
||||
|
||||
|
||||
def test_fastvideo_metadata_builds_release_sampling_metadata():
|
||||
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
|
||||
assert spec.vsa_sparsity == 0.8
|
||||
assert spec.release_metadata() == {
|
||||
"partition": "fl2va",
|
||||
"tasks": ["t2va"],
|
||||
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
|
||||
"base_schedule": list(FASTH3_V2_BASE_SCHEDULE.base_schedule),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value"),
|
||||
[
|
||||
("model_id", "FastVideo/FastH3-other"),
|
||||
("schema_version", "fasth3-inference-contract-v0"),
|
||||
("dmd_denoising_steps", [999, 749, 500, 250]),
|
||||
("transformer_forwards", 4),
|
||||
("video_scheduler_shift", 12.0),
|
||||
("audio_scheduler_shift", 4.0),
|
||||
("guidance_scale", 2.0),
|
||||
("attention_backend", "FLASH_ATTN"),
|
||||
("vsa_sparsity", 0.9),
|
||||
("vsa_tile_size", 32),
|
||||
("task", "t2va"),
|
||||
],
|
||||
)
|
||||
def test_fastvideo_metadata_rejects_contract_drift(field, value):
|
||||
metadata = _fastvideo_metadata()
|
||||
metadata[field] = value
|
||||
|
||||
with pytest.raises(ValueError, match=field):
|
||||
FastH3CheckpointSpec.from_metadata(metadata)
|
||||
|
||||
|
||||
def test_fastvideo_metadata_rejects_missing_contract_field():
|
||||
metadata = _fastvideo_metadata()
|
||||
metadata.pop("task")
|
||||
|
||||
with pytest.raises(ValueError, match="task"):
|
||||
FastH3CheckpointSpec.from_metadata(metadata)
|
||||
|
||||
|
||||
def test_legacy_h3_metadata_is_not_treated_as_fastvideo_release():
|
||||
with pytest.raises(ValueError, match="model_id"):
|
||||
FastH3CheckpointSpec.from_metadata(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"partition": "fl2va",
|
||||
"tasks": ["t2va", "fl2va"],
|
||||
"sigma_shift_scales": {"video": 12.0, "audio": 3.0},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_v2_serving_contract_accepts_backend_selected_for_h3_self_role():
|
||||
config = _od_config(
|
||||
backend="TORCH_SDPA",
|
||||
per_role={"self": AttentionSpec(backend="FASTVIDEO_VSA")},
|
||||
)
|
||||
FastH3CheckpointSpec.from_metadata(_fastvideo_metadata()).check_serving_contract(
|
||||
partition="fl2va", od_config=config
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "partition", "match"),
|
||||
[
|
||||
({}, "ref2va", "task-type fl2va"),
|
||||
({"lora_path": "/tmp/adapter"}, "fl2va", "LoRA"),
|
||||
({"backend": "TORCH_SDPA"}, "fl2va", "FASTVIDEO_VSA"),
|
||||
({"topk": 2}, "fl2va", "fixed fastvideo_vsa_topk"),
|
||||
({"ring_degree": 2}, "fl2va", "local attention"),
|
||||
({"allgather_degree": 2}, "fl2va", "local attention"),
|
||||
],
|
||||
)
|
||||
def test_v2_serving_contract_rejects_incompatible_runtime(kwargs, partition, match):
|
||||
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
with pytest.raises(ValueError, match=match):
|
||||
spec.check_serving_contract(partition=partition, od_config=_od_config(**kwargs))
|
||||
|
||||
|
||||
def test_v2_serving_contract_reads_a_per_role_fixed_topk_override():
|
||||
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
config = _od_config(
|
||||
per_role={"self": AttentionSpec(backend="FASTVIDEO_VSA", fastvideo_vsa_topk=1)},
|
||||
)
|
||||
with pytest.raises(ValueError, match="fixed fastvideo_vsa_topk"):
|
||||
spec.check_serving_contract(partition="fl2va", od_config=config)
|
||||
|
||||
|
||||
def test_v2_request_contract_accepts_omitted_steps_and_default_shifts():
|
||||
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
spec.check_request(_sampling(num_inference_steps=None))
|
||||
spec.check_request(_sampling(extra_args={"flow_shift": 10, "audio_flow_shift": 3}, guidance_scale=1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sampling",
|
||||
[
|
||||
_sampling(num_inference_steps=None),
|
||||
_sampling(timesteps=torch.tensor([0.9, 0.5, 0.0])),
|
||||
_sampling(sigmas=[1.0, 0.5, 0.0]),
|
||||
],
|
||||
)
|
||||
def test_v2_step_execution_requires_pinned_count_and_schedule(sampling):
|
||||
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
with pytest.raises(OmniClientError, match="step execution"):
|
||||
spec.check_request(sampling, step_execution=True)
|
||||
|
||||
|
||||
def test_v2_step_execution_accepts_only_explicit_pinned_count():
|
||||
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
spec.check_request(_sampling(num_inference_steps=8), step_execution=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "match"),
|
||||
[
|
||||
({"num_inference_steps": 7}, "num_inference_steps=8"),
|
||||
({"num_inference_steps": 9}, "num_inference_steps=8"),
|
||||
({"num_inference_steps": "8"}, "num_inference_steps=8"),
|
||||
({"extra_args": {"flow_shift": 9}}, "flow_shift=10"),
|
||||
({"extra_args": {"audio_flow_shift": 4}}, "audio_flow_shift=3"),
|
||||
({"extra_args": {"flow_shift": "bad"}}, "flow_shift=10"),
|
||||
({"guidance_scale": 2.0}, "guidance_scale=1"),
|
||||
(
|
||||
{"lora_request": LoRARequest(lora_name="test", lora_int_id=1, lora_path="/tmp/test.safetensors")},
|
||||
"per-request LoRA",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_v2_request_contract_rejects_sampling_drift(kwargs, match):
|
||||
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
with pytest.raises(OmniClientError, match=match):
|
||||
spec.check_request(_sampling(**kwargs))
|
||||
|
||||
|
||||
def test_resolve_native_vaes_uses_pinned_revision_and_only_vae_patterns(tmp_path, monkeypatch):
|
||||
model_root = tmp_path / "FastH3"
|
||||
model_root.mkdir()
|
||||
revision = "base-revision-123"
|
||||
(model_root / "provenance.json").write_text(
|
||||
json.dumps({"base_model": f"hf://{FASTH3_BASE_MODEL}@{revision}"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
snapshot = tmp_path / "hf-snapshot"
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
def fake_download(**kwargs: object) -> str:
|
||||
calls.update(kwargs)
|
||||
snapshot.mkdir()
|
||||
return str(snapshot)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.models.minimax_h3.fasth3_checkpoint.download_weights_from_hf_specific",
|
||||
fake_download,
|
||||
)
|
||||
|
||||
result = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata()).resolve_native_vaes(model_root)
|
||||
|
||||
assert result == snapshot / "FL2VA"
|
||||
assert calls == {
|
||||
"model_name_or_path": FASTH3_BASE_MODEL,
|
||||
"cache_dir": None,
|
||||
"allow_patterns": ["FL2VA/video_vae/**", "FL2VA/audio_vae/**"],
|
||||
"revision": revision,
|
||||
"require_all": True,
|
||||
}
|
||||
assert sorted(path.name for path in model_root.iterdir()) == ["provenance.json"]
|
||||
assert not (model_root / "model_index.json").exists()
|
||||
assert not (model_root / "conversion.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_model",
|
||||
[
|
||||
"MiniMaxAI/MiniMax-H3@main",
|
||||
"hf://MiniMaxAI/MiniMax-H3@",
|
||||
"hf://Other/MiniMax-H3@revision",
|
||||
"",
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_resolve_native_vaes_rejects_unpinned_provenance(tmp_path, base_model):
|
||||
model_root = tmp_path / "FastH3"
|
||||
model_root.mkdir()
|
||||
(model_root / "provenance.json").write_text(json.dumps({"base_model": base_model}), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="provenance"):
|
||||
FastH3CheckpointSpec.from_metadata(_fastvideo_metadata()).resolve_native_vaes(model_root)
|
||||
|
||||
|
||||
def test_modular_local_model_root_uses_text_encoder_without_partition_files(tmp_path):
|
||||
root = tmp_path / "FastH3"
|
||||
(root / "text_encoder").mkdir(parents=True)
|
||||
(root / "text_encoder" / "config.json").write_text("{}", encoding="utf-8")
|
||||
(root / "modular_model_index.json").write_text(
|
||||
json.dumps({"_class_name": "MiniMaxH3ModularPipeline"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert is_minimax_h3_modular(str(root)) is True
|
||||
assert resolve_minimax_h3_model_root(str(root), None, "fl2va") == str(root / "text_encoder")
|
||||
assert not (root / "FL2VA").exists()
|
||||
|
||||
|
||||
def test_modular_hub_model_root_downloads_only_text_encoder(monkeypatch, tmp_path):
|
||||
from vllm_omni.model_executor.models.minimax_h3 import checkpoint as checkpoint_module
|
||||
|
||||
snapshot = tmp_path / "modular-snapshot"
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
def fake_index(model: str, *, revision: str | None = None) -> dict[str, str]:
|
||||
assert model == "FastVideo/FastVideo-FastH3-8-Step-V2"
|
||||
assert revision == "release-revision"
|
||||
return {"_class_name": "MiniMaxH3ModularPipeline"}
|
||||
|
||||
def fake_download(**kwargs: object) -> str:
|
||||
calls.update(kwargs)
|
||||
snapshot.mkdir()
|
||||
return str(snapshot)
|
||||
|
||||
monkeypatch.setattr(checkpoint_module, "get_diffusion_model_index", fake_index)
|
||||
monkeypatch.setattr(checkpoint_module, "download_weights_from_hf_specific", fake_download)
|
||||
|
||||
result = resolve_minimax_h3_model_root(
|
||||
"FastVideo/FastVideo-FastH3-8-Step-V2",
|
||||
"release-revision",
|
||||
"fl2va",
|
||||
)
|
||||
|
||||
assert result == str(snapshot / "text_encoder")
|
||||
assert calls == {
|
||||
"model_name_or_path": "FastVideo/FastVideo-FastH3-8-Step-V2",
|
||||
"cache_dir": None,
|
||||
"allow_patterns": ["text_encoder/**"],
|
||||
"revision": "release-revision",
|
||||
"require_all": True,
|
||||
}
|
||||
|
||||
|
||||
def test_native_hub_model_with_modular_class_keeps_partition_layout(monkeypatch):
|
||||
from vllm_omni.model_executor.models.minimax_h3 import checkpoint as checkpoint_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
checkpoint_module,
|
||||
"get_diffusion_model_index",
|
||||
lambda _model, *, revision=None: {"_class_name": "MiniMaxH3ModularPipeline"},
|
||||
)
|
||||
|
||||
assert checkpoint_module.is_minimax_h3_modular("MiniMaxAI/MiniMax-H3") is False
|
||||
|
||||
|
||||
def test_native_local_model_root_keeps_partition_layout(tmp_path):
|
||||
root = tmp_path / "MiniMax-H3"
|
||||
(root / "FL2VA" / "text_encoder").mkdir(parents=True)
|
||||
(root / "FL2VA" / "text_encoder" / "config.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
assert is_minimax_h3_modular(str(root)) is False
|
||||
assert resolve_minimax_h3_model_root(str(root), None, "fl2va") == str(root / "FL2VA" / "text_encoder")
|
||||
|
||||
|
||||
def test_diffusion_resolver_returns_local_modular_root_without_partition_files(tmp_path):
|
||||
root = tmp_path / "FastH3"
|
||||
root.mkdir()
|
||||
(root / "modular_model_index.json").write_text(
|
||||
json.dumps({"_class_name": "MiniMaxH3ModularPipeline"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert resolve_minimax_h3_diffusion_model_path(str(root), None, "fl2va") == str(root)
|
||||
assert not (root / "FL2VA").exists()
|
||||
assert not (root / "Ref2VA").exists()
|
||||
|
||||
|
||||
def test_diffusion_resolver_hub_modular_downloads_transformer_contract_only(monkeypatch, tmp_path):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as pipeline_module
|
||||
|
||||
snapshot = tmp_path / "modular-snapshot"
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
def fake_modular(_model: str, _revision: str | None = None) -> bool:
|
||||
return True
|
||||
|
||||
def fake_download(**kwargs: object) -> str:
|
||||
calls.update(kwargs)
|
||||
snapshot.mkdir()
|
||||
return str(snapshot)
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "is_minimax_h3_modular", fake_modular)
|
||||
monkeypatch.setattr(pipeline_module, "download_weights_from_hf_specific", fake_download)
|
||||
|
||||
result = resolve_minimax_h3_diffusion_model_path(
|
||||
"FastVideo/FastVideo-FastH3-8-Step-V2", "release-revision", "fl2va"
|
||||
)
|
||||
|
||||
assert result == str(snapshot)
|
||||
assert calls == {
|
||||
"model_name_or_path": "FastVideo/FastVideo-FastH3-8-Step-V2",
|
||||
"cache_dir": None,
|
||||
"allow_patterns": [
|
||||
"modular_model_index.json",
|
||||
"fastvideo_inference.json",
|
||||
"provenance.json",
|
||||
"transformer/**",
|
||||
],
|
||||
"revision": "release-revision",
|
||||
"require_all": True,
|
||||
}
|
||||
patterns = calls["allow_patterns"]
|
||||
assert isinstance(patterns, list)
|
||||
assert all("video_vae" not in pattern and "audio_vae" not in pattern for pattern in patterns)
|
||||
|
||||
|
||||
def test_diffusers_arch_config_normalizes_all_aliases():
|
||||
arch = MiniMaxH3DiTArchConfig.from_mapping(
|
||||
{
|
||||
"num_refiner_layers": 3,
|
||||
"ffn_dim": 12,
|
||||
"in_channels": 4,
|
||||
"audio_in_channels": 6,
|
||||
"freq_dim": 8,
|
||||
"time_embed_hidden_dim": 10,
|
||||
"rope_freq_dim": 5,
|
||||
"patch_size": [1, 2, 2],
|
||||
}
|
||||
)
|
||||
|
||||
assert arch.token_refiner_num_layers == 3
|
||||
assert arch.ffn_hidden_size == 12
|
||||
assert arch.latents_dim == 4
|
||||
assert arch.audio_latents_dim == 6
|
||||
assert arch.timestep_input_dim == 8
|
||||
assert arch.time_embed_hidden_size == 10
|
||||
assert arch.rope_inv_freq_len == 5
|
||||
assert arch.patch_size == (1, 2, 2)
|
||||
|
||||
|
||||
class _WeightTarget(nn.Module):
|
||||
def __init__(self, shape: tuple[int, ...], loader: Callable[..., None]) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(shape))
|
||||
self.weight.weight_loader = loader
|
||||
|
||||
|
||||
class _PipelineLoadTransformer(nn.Module):
|
||||
def __init__(self, loaded: set[str]) -> None:
|
||||
super().__init__()
|
||||
self.arch = MiniMaxH3DiTArchConfig(num_layers=1)
|
||||
self._loaded = loaded
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
list(weights)
|
||||
return self._loaded
|
||||
|
||||
def post_load_weights(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def test_pipeline_load_weights_rejects_missing_fast_h3_compression_gate():
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.transformer = _PipelineLoadTransformer({"blocks.0.attn.qkv_proj.weight"})
|
||||
pipeline.video_vae = None
|
||||
pipeline.audio_vae = None
|
||||
pipeline._fasth3 = None
|
||||
pipeline._fasth3_checkpoint = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
|
||||
with pytest.raises(ValueError, match="missing compression gates"):
|
||||
pipeline.load_weights([("transformer.blocks.0.attn.qkv_proj.weight", torch.zeros(1))])
|
||||
|
||||
|
||||
def _load_fixture(
|
||||
*, diffusers_weights: bool
|
||||
) -> tuple[MiniMaxH3DiTModel, list[tuple[str | None, torch.Tensor]], list[tuple[int, torch.Tensor]]]:
|
||||
model = object.__new__(MiniMaxH3DiTModel)
|
||||
nn.Module.__init__(model)
|
||||
model.arch = MiniMaxH3DiTArchConfig(
|
||||
num_layers=1,
|
||||
hidden_size=3,
|
||||
num_attention_heads=2,
|
||||
attention_head_dim=2,
|
||||
ffn_hidden_size=6,
|
||||
rope_inv_freq_len=2,
|
||||
)
|
||||
model._diffusers_weights = diffusers_weights
|
||||
model._rope_theta = 10000.0
|
||||
qkv_calls: list[tuple[str | None, torch.Tensor]] = []
|
||||
fc1_calls: list[tuple[int, torch.Tensor]] = []
|
||||
|
||||
def qkv_loader(param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str | None = None) -> None:
|
||||
if shard_id is None:
|
||||
assert loaded_weight.shape == (12, 3)
|
||||
param.data.copy_(loaded_weight)
|
||||
qkv_calls.append((None, loaded_weight.clone()))
|
||||
return
|
||||
assert loaded_weight.shape == (4, 3)
|
||||
assert shard_id in {"q", "k", "v"}
|
||||
offset = {"q": 0, "k": 4, "v": 8}[shard_id]
|
||||
param.data[offset : offset + 4].copy_(loaded_weight)
|
||||
qkv_calls.append((shard_id, loaded_weight.clone()))
|
||||
|
||||
def fc1_loader(param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: int) -> None:
|
||||
assert loaded_weight.shape == (3, 3)
|
||||
param.data[shard_id * 3 : (shard_id + 1) * 3].copy_(loaded_weight)
|
||||
fc1_calls.append((shard_id, loaded_weight.clone()))
|
||||
|
||||
model.blocks = nn.ModuleList([nn.Module()])
|
||||
model.blocks[0].attn = nn.Module()
|
||||
model.blocks[0].attn.qkv_proj = _WeightTarget((12, 3), qkv_loader)
|
||||
model.blocks[0].mlp = nn.Module()
|
||||
model.blocks[0].mlp.fc1 = _WeightTarget((6, 3), fc1_loader)
|
||||
model.rope = nn.Module()
|
||||
model.rope.register_buffer("inv_freq", torch.zeros(2))
|
||||
return model, qkv_calls, fc1_calls
|
||||
|
||||
|
||||
def test_diffusers_load_weights_uses_shape_aware_qkv_and_mlp_slices():
|
||||
model, qkv_calls, fc1_calls = _load_fixture(diffusers_weights=True)
|
||||
q = torch.arange(12, dtype=torch.float32).reshape(4, 3) + 10
|
||||
k = torch.arange(12, dtype=torch.float32).reshape(4, 3) + 100
|
||||
v = torch.arange(12, dtype=torch.float32).reshape(4, 3) + 200
|
||||
mlp = torch.arange(18, dtype=torch.float32).reshape(6, 3) + 300
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("transformer_blocks.0.attn.to_q.weight", q),
|
||||
("transformer_blocks.0.attn.to_k.weight", k),
|
||||
("transformer_blocks.0.attn.to_v.weight", v),
|
||||
("transformer_blocks.0.ff.net.0.proj.weight", mlp),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {
|
||||
"blocks.0.attn.qkv_proj.weight",
|
||||
"blocks.0.mlp.fc1.weight",
|
||||
"rope.inv_freq",
|
||||
}
|
||||
assert [shard_id for shard_id, _ in qkv_calls] == ["q", "k", "v"]
|
||||
assert [shard_id for shard_id, _ in fc1_calls] == [0, 1]
|
||||
torch.testing.assert_close(model.blocks[0].attn.qkv_proj.weight, torch.cat((q, k, v)))
|
||||
torch.testing.assert_close(model.blocks[0].mlp.fc1.weight, torch.cat((mlp[3:], mlp[:3])))
|
||||
torch.testing.assert_close(
|
||||
model.rope.inv_freq,
|
||||
1.0 / (10000.0 ** (torch.arange(0, 4, 2, dtype=torch.float32) / 4)),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
def test_native_load_weights_keeps_grouped_qkv_and_gate_first_layouts():
|
||||
model, qkv_calls, fc1_calls = _load_fixture(diffusers_weights=False)
|
||||
grouped_qkv = torch.arange(36, dtype=torch.float32).reshape(12, 3)
|
||||
fc1 = torch.arange(18, dtype=torch.float32).reshape(6, 3)
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("blocks.0.attn.qkv_proj.weight", grouped_qkv),
|
||||
("blocks.0.mlp.fc1.weight", fc1),
|
||||
]
|
||||
)
|
||||
|
||||
expected_qkv = torch.cat(
|
||||
(
|
||||
grouped_qkv[:2],
|
||||
grouped_qkv[6:8],
|
||||
grouped_qkv[2:4],
|
||||
grouped_qkv[8:10],
|
||||
grouped_qkv[4:6],
|
||||
grouped_qkv[10:],
|
||||
)
|
||||
)
|
||||
assert loaded == {"blocks.0.attn.qkv_proj.weight", "blocks.0.mlp.fc1.weight"}
|
||||
assert [shard_id for shard_id, _ in qkv_calls] == [None]
|
||||
assert [shard_id for shard_id, _ in fc1_calls] == [0, 1]
|
||||
torch.testing.assert_close(model.blocks[0].attn.qkv_proj.weight, expected_qkv)
|
||||
torch.testing.assert_close(model.blocks[0].mlp.fc1.weight, fc1)
|
||||
assert torch.equal(model.rope.inv_freq, torch.zeros(2))
|
||||
|
||||
|
||||
def test_diffusers_load_weights_rejects_unknown_name():
|
||||
model, _, _ = _load_fixture(diffusers_weights=True)
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported Diffusers H3 weight"):
|
||||
model.load_weights([("transformer_blocks.0.attn.not_a_projection.weight", torch.zeros(4, 3))])
|
||||
|
||||
|
||||
def test_diffusers_load_weights_rejects_duplicate_source_name():
|
||||
model, _, _ = _load_fixture(diffusers_weights=True)
|
||||
weight = torch.zeros(4, 3)
|
||||
|
||||
with pytest.raises(ValueError, match="duplicate Diffusers H3 weight"):
|
||||
model.load_weights(
|
||||
[
|
||||
("transformer_blocks.0.attn.to_q.weight", weight),
|
||||
("transformer_blocks.0.attn.to_q.weight", weight),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_diffusers_load_weights_rejects_incomplete_qkv_group():
|
||||
model, _, _ = _load_fixture(diffusers_weights=True)
|
||||
|
||||
with pytest.raises(ValueError, match="incomplete Diffusers H3 QKV"):
|
||||
model.load_weights(
|
||||
[
|
||||
("transformer_blocks.0.attn.to_q.weight", torch.zeros(4, 3)),
|
||||
("transformer_blocks.0.attn.to_k.weight", torch.zeros(4, 3)),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_diffusers_load_weights_rejects_missing_compression_gate_parameter():
|
||||
model, _, _ = _load_fixture(diffusers_weights=True)
|
||||
|
||||
with pytest.raises(ValueError, match="has no model parameter"):
|
||||
model.load_weights([("transformer_blocks.0.attn.to_gate_compress.weight", torch.zeros(4, 3))])
|
||||
|
||||
|
||||
def test_resolve_sigma_positions_uses_release_metadata_schedule():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline._fasth3 = None
|
||||
pipeline._lora_sigma_schedules = {}
|
||||
spec = FastH3CheckpointSpec.from_metadata(_fastvideo_metadata())
|
||||
pipeline._base_schedule_by_partition = {
|
||||
"fl2va": DMD2SigmaSchedule.from_metadata(spec.release_metadata()),
|
||||
}
|
||||
|
||||
positions, steps = pipeline._resolve_sigma_positions("t2va", _sampling())
|
||||
assert positions == FASTH3_V2_BASE_SCHEDULE.base_schedule
|
||||
assert steps == 8 == len(positions) - 1
|
||||
with pytest.raises(OmniClientError, match="must be 8"):
|
||||
pipeline._resolve_sigma_positions("t2va", _sampling(num_inference_steps=9))
|
||||
@@ -87,7 +87,8 @@ def _collective_worker(rank, rendezvous, backend):
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=32, width=32, num_frames=96, extra_args={"task": "t2va", "aspect_ratio": "1:1"}
|
||||
)
|
||||
text_conditioning = model._prepare_local_conditioning({"prompt": "test"}, sampling)
|
||||
text_conditioning, window_text = model._prepare_local_conditioning({"prompt": "test"}, sampling)
|
||||
assert window_text is None
|
||||
torch.testing.assert_close(text_conditioning.hidden_states, torch.full_like(text_conditioning.hidden_states, 2))
|
||||
assert text_conditioning.visual_condition is None
|
||||
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.continuation import (
|
||||
diffuse_continuation,
|
||||
plan_continuation_windows,
|
||||
resolve_continuation,
|
||||
)
|
||||
from vllm_omni.diffusion.models.minimax_h3.packed_sequence import minimax_h3_packed_sequence_ref2va_blocks
|
||||
from vllm_omni.errors import OmniClientError
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("vary_prompts", [False, True])
|
||||
@pytest.mark.parametrize("total,window", [(1807, 277), (1807, 243), (124, 277)])
|
||||
def test_continuation_retains_prefix_and_global_stereo_timeline(total, window, vary_prompts):
|
||||
audio_t = round(total * 40 / 24)
|
||||
source = torch.arange(2 * audio_t * 32).reshape(2, audio_t, 32).float()
|
||||
calls = []
|
||||
plan = plan_continuation_windows(total, window, 22)
|
||||
texts = [(torch.full((3 + i, 8), float(i)), torch.ones(3 + i, dtype=torch.long)) for i in range(len(plan))]
|
||||
|
||||
def sample(**kw):
|
||||
calls.append(kw)
|
||||
t = kw["latent_t"]
|
||||
if vary_prompts:
|
||||
assert kw["text_embeddings"] is texts[len(calls) - 1][0]
|
||||
assert kw["text_tags"] is texts[len(calls) - 1][1]
|
||||
assert kw["media_time_origin"] == len(texts[-1][0])
|
||||
# Different constants expose replacement of an old prefix or an
|
||||
# accidental append of the hidden overlap.
|
||||
video = torch.full((1, 24, t, 2, 2), float(len(calls)))
|
||||
audio = kw["locked_audio_rows"].reshape(2, kw["audio_t"], 32).permute(0, 2, 1)
|
||||
if len(calls) > 1:
|
||||
assert torch.all(kw["visual_condition"] == len(calls) - 1)
|
||||
assert kw["ref_blocks"][-1]["kind"] == "latent_guide"
|
||||
return video, audio
|
||||
|
||||
kwargs = dict(
|
||||
num_frames=total,
|
||||
latent_t=(total - 5) // 17 * 5 + 2,
|
||||
latent_h=2,
|
||||
latent_w=2,
|
||||
audio_t=audio_t,
|
||||
locked_audio_rows=source.reshape(-1, 32),
|
||||
)
|
||||
video, audio = diffuse_continuation(
|
||||
sample, kwargs, window_frames=window, overlap_frames=22, text_conditioning=texts if vary_prompts else None
|
||||
)
|
||||
torch.testing.assert_close(audio, source.permute(0, 2, 1))
|
||||
plan = plan_continuation_windows(total, window, 22)
|
||||
assert [call["temporal_offset"] for call in calls] == pytest.approx([part.start * 40 / 24 for part in plan])
|
||||
offset = 0
|
||||
for i, part in enumerate(plan):
|
||||
new_frames = part.end - part.start - part.overlap
|
||||
count = (new_frames - 5) // 17 * 5 + 2 if i == 0 else new_frames // 17 * 5
|
||||
assert torch.all(video[:, :, offset : offset + count] == i + 1)
|
||||
offset += count
|
||||
assert offset == video.shape[2] == kwargs["latent_t"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("temporal_offset", [0.0, 425.0, 221 * 40 / 24])
|
||||
def test_latent_guide_shares_target_clock_and_remains_condition_only(temporal_offset):
|
||||
packed = minimax_h3_packed_sequence_ref2va_blocks(
|
||||
temporal_offset=temporal_offset,
|
||||
text_len=3,
|
||||
latent_t=12,
|
||||
latent_h=4,
|
||||
latent_w=4,
|
||||
audio_t=68,
|
||||
ref_blocks=[
|
||||
{"kind": "image", "latent_h": 4, "latent_w": 4},
|
||||
{"kind": "latent_guide", "latent_t": 7, "latent_h": 4, "latent_w": 4, "ref_audio_t": 37},
|
||||
],
|
||||
)
|
||||
pos = packed["img_position_ids"]
|
||||
visual = packed["img_pos"]
|
||||
target = visual[packed["update_mask"]]
|
||||
guide = visual[~packed["update_mask"]][4:]
|
||||
torch.testing.assert_close(pos[guide], pos[target[:28]])
|
||||
audio = packed["audio_pos"]
|
||||
guide_audio = audio[~packed["audio_update_mask"]].reshape(2, 37)
|
||||
target_audio = audio[packed["audio_update_mask"]].reshape(2, 68)
|
||||
torch.testing.assert_close(pos[guide_audio], pos[target_audio[:, :37]])
|
||||
assert pos[target[0], 0] == pytest.approx(4 + temporal_offset)
|
||||
torch.testing.assert_close(pos[:3, 0], torch.arange(3, dtype=torch.float64))
|
||||
assert torch.all(pos[3:7, 0] == 3) # static reference image stays fixed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra,step",
|
||||
[
|
||||
({"long_video_mode": "bad"}, False),
|
||||
({"long_video_mode": "continuation"}, True),
|
||||
({"long_video_mode": "continuation", "continuation_overlap_frames": 23}, False),
|
||||
({"long_video_mode": "continuation", "continuation_window_frames": True}, False),
|
||||
({"long_video_mode": "continuation", "continuation_overlap_frames": 277}, False),
|
||||
],
|
||||
)
|
||||
def test_invalid_continuation_options(extra, step):
|
||||
with pytest.raises(OmniClientError):
|
||||
resolve_continuation(extra, task="ref2va", step_execution=step)
|
||||
|
||||
|
||||
def test_global_offset_moves_temporal_references_but_not_images_or_padding():
|
||||
kwargs = dict(
|
||||
text_len=3,
|
||||
latent_t=12,
|
||||
latent_h=4,
|
||||
latent_w=4,
|
||||
audio_t=68,
|
||||
ref_blocks=[
|
||||
{"kind": "image", "latent_h": 4, "latent_w": 4},
|
||||
{"kind": "audio", "ref_audio_t": 3},
|
||||
{"kind": "video_audio", "ref_audio_t": 2, "latent_t": 2, "latent_h": 4, "latent_w": 4},
|
||||
{"kind": "image", "latent_h": 4, "latent_w": 4},
|
||||
{"kind": "latent_guide", "ref_audio_t": 37, "latent_t": 7, "latent_h": 4, "latent_w": 4},
|
||||
],
|
||||
)
|
||||
base = minimax_h3_packed_sequence_ref2va_blocks(**kwargs)
|
||||
shifted = minimax_h3_packed_sequence_ref2va_blocks(**kwargs, temporal_offset=425.0)
|
||||
moving = base["image_mask"] | base["audio_mask"]
|
||||
moving[3:7] = False # first static image
|
||||
moving[25:29] = False # static image after temporal references
|
||||
delta = shifted["img_position_ids"] - base["img_position_ids"]
|
||||
torch.testing.assert_close(delta[:, 0], moving.double() * 425)
|
||||
assert torch.count_nonzero(delta[:, 1:]) == 0
|
||||
for key, value in base.items():
|
||||
if isinstance(value, torch.Tensor) and key != "img_position_ids":
|
||||
torch.testing.assert_close(shifted[key], value)
|
||||
|
||||
|
||||
def test_long_ref2va_defaults_to_continuation_with_explicit_full_opt_out():
|
||||
assert resolve_continuation({"long_video": True}, task="ref2va", step_execution=False) == (277, 22)
|
||||
assert resolve_continuation({}, task="ref2va", step_execution=False) is None
|
||||
assert (
|
||||
resolve_continuation({"long_video": True, "long_video_mode": "full"}, task="ref2va", step_execution=False)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_different_chunk_prompt_lengths_keep_the_same_media_clock():
|
||||
kwargs = dict(
|
||||
latent_t=7,
|
||||
latent_h=4,
|
||||
latent_w=4,
|
||||
audio_t=37,
|
||||
ref_blocks=[{"kind": "image", "latent_h": 4, "latent_w": 4}],
|
||||
media_time_origin=20,
|
||||
)
|
||||
first = minimax_h3_packed_sequence_ref2va_blocks(text_len=3, **kwargs)
|
||||
second = minimax_h3_packed_sequence_ref2va_blocks(text_len=15, temporal_offset=425, **kwargs)
|
||||
for key, mask in (("img_pos", "update_mask"), ("audio_pos", "audio_update_mask")):
|
||||
a = first["img_position_ids"][first[key][first[mask]]]
|
||||
b = second["img_position_ids"][second[key][second[mask]]]
|
||||
torch.testing.assert_close(b[:, 0] - a[:, 0], torch.full_like(a[:, 0], 425))
|
||||
torch.testing.assert_close(b[:, 1:], a[:, 1:])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_count", [None, 7, 6])
|
||||
def test_request_encodes_only_window_prompts_and_prepares_references_once(monkeypatch, prompt_count):
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3EncoderMediaConditioning
|
||||
|
||||
pipeline = object.__new__(module.MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.load_text_encoder = True
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.od_config = None
|
||||
pipeline._active_turbo_spec = lambda sampling: None
|
||||
pipeline._has_active_native_lora = lambda sampling: False
|
||||
pipeline._resolve_task = lambda *args, **kwargs: "ref2va"
|
||||
encoded, prepared_inputs, encoded_media = [], [], []
|
||||
prepare = module.prepare_encoder_inputs
|
||||
|
||||
def prepare_once(*args, **kwargs):
|
||||
result = prepare(*args, **kwargs)
|
||||
prepared_inputs.append(result)
|
||||
return result
|
||||
|
||||
def encode_text(prepared):
|
||||
encoded.append(prepared)
|
||||
return torch.full((3, 5120), len(encoded), dtype=torch.bfloat16), torch.zeros(3, dtype=torch.long)
|
||||
|
||||
def encode_media(media):
|
||||
encoded_media.append(media)
|
||||
return MiniMaxH3EncoderMediaConditioning(
|
||||
task=media.task,
|
||||
height=media.height,
|
||||
width=media.width,
|
||||
num_frames=media.num_frames,
|
||||
latent_t=media.latent_t,
|
||||
audio_t=media.audio_t,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(module, "prepare_encoder_inputs", prepare_once)
|
||||
pipeline.encode_prompt = encode_text
|
||||
pipeline._encode_local_media = encode_media
|
||||
pipeline._prepare_encoder_conditioning_inputs = lambda conditioning, sampling: {
|
||||
"text_embeddings": conditioning.hidden_states,
|
||||
"continuation": (277, 22),
|
||||
"num_frames": conditioning.num_frames,
|
||||
"task": conditioning.task,
|
||||
}
|
||||
extra = {"task": "ref2va", "long_video": True, "duration": 75}
|
||||
if prompt_count is not None:
|
||||
extra["continuation_prompts"] = [f"window {i}" for i in range(prompt_count)]
|
||||
sampling = OmniDiffusionSamplingParams(width=64, height=64, fps=24, extra_args=extra)
|
||||
request = {"prompt": "main prompt", "multi_modal_data": {"image": Image.new("RGB", (256, 256))}}
|
||||
if prompt_count == 6:
|
||||
with pytest.raises(OmniClientError, match="exactly 7"):
|
||||
pipeline._prepare_request_inputs(request, sampling)
|
||||
assert not encoded and not encoded_media
|
||||
return
|
||||
context = pipeline._prepare_request_inputs(request, sampling)
|
||||
assert [item.prompt for item in encoded] == extra.get("continuation_prompts", ["main prompt"])
|
||||
assert len(prepared_inputs) == len(encoded_media) == 1
|
||||
assert all(item.media is prepared_inputs[0].media for item in encoded)
|
||||
if prompt_count is not None:
|
||||
texts = context["continuation_text_conditioning"]
|
||||
assert len(texts) == 7
|
||||
assert texts[0][0] is context["text_embeddings"]
|
||||
assert [int(hidden[0, 0]) for hidden, _ in texts] == list(range(1, 8))
|
||||
else:
|
||||
assert "continuation_text_conditioning" not in context
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import json
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from multiprocessing.reduction import ForkingPickler
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, TypeVar
|
||||
@@ -282,15 +282,44 @@ def test_encoder_task_validation_and_transformer_routing():
|
||||
assert pipeline._resolve_task(None, {"image": object()}) == "fl2va"
|
||||
assert pipeline._resolve_task(None, {"audio": object()}) == "ref2va"
|
||||
assert pipeline._resolve_task(None, {"video": object()}) == "ref2va"
|
||||
assert pipeline._resolve_task(None, {"audio": object()}, audio_mode="lock_source") == "t2va"
|
||||
assert (
|
||||
pipeline._resolve_task(
|
||||
None,
|
||||
{"image": object(), "audio": object()},
|
||||
audio_mode="lock_source",
|
||||
)
|
||||
== "fl2va"
|
||||
)
|
||||
assert (
|
||||
pipeline._resolve_task(
|
||||
None,
|
||||
{"video": object(), "audio": object()},
|
||||
audio_mode="lock_source",
|
||||
)
|
||||
== "ref2va"
|
||||
)
|
||||
fl2v_spec = _turbo_spec("minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors")
|
||||
ref2v_spec = _turbo_spec("minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors")
|
||||
assert pipeline._resolve_task("fl2va", {"image": object()}, turbo_spec=fl2v_spec) == "fl2va"
|
||||
with pytest.raises(OmniClientError, match="serves"):
|
||||
pipeline._resolve_task("ref2va", {}, turbo_spec=fl2v_spec)
|
||||
assert pipeline._resolve_task("ref2va", {"image": object()}, turbo_spec=ref2v_spec) == "ref2va"
|
||||
pipeline.partition = "fl2va"
|
||||
pipeline.supported_tasks = frozenset({"t2va", "fl2va"})
|
||||
assert pipeline._resolve_task(None, {"audio": object()}, audio_mode="lock_source") == "t2va"
|
||||
assert (
|
||||
pipeline._resolve_task(
|
||||
None,
|
||||
{"image": object(), "audio": object()},
|
||||
audio_mode="lock_source",
|
||||
)
|
||||
== "fl2va"
|
||||
)
|
||||
pipeline.partition = "ref2va"
|
||||
pipeline.supported_tasks = frozenset({"ref2va"})
|
||||
assert pipeline._resolve_task("ref2va") == "ref2va"
|
||||
assert pipeline._resolve_task(None, {"audio": object()}, audio_mode="lock_source") == "ref2va"
|
||||
|
||||
pipeline.partition = "combined"
|
||||
pipeline.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
|
||||
@@ -423,6 +452,7 @@ def test_pipeline_loads_task_selected_components_with_encoder_ownership(
|
||||
assert pipeline.partition == expected_partition
|
||||
assert pipeline.supported_tasks == expected_tasks
|
||||
assert len(created["dit"]) == expected_dits
|
||||
assert all(kwargs["diffusers_weights"] is False for _, kwargs in created["dit"])
|
||||
component_path = tmp_path / component_partition
|
||||
assert created["video_vae"] == [str(component_path / "video_vae")]
|
||||
assert created["audio_vae"] == [str(component_path / "audio_vae")]
|
||||
@@ -1041,6 +1071,56 @@ def test_packed_attention_rejects_backends_without_multi_doc_capability(backend_
|
||||
)
|
||||
|
||||
|
||||
def test_rainfusion_packed_padding_stays_mask_free_on_unaligned_lengths():
|
||||
"""Regression for the #5543 follow-up crash fixed in #7235.
|
||||
|
||||
MiniMax-H3 pads every packed request to a multiple of 64 rows, so any
|
||||
non-64-aligned length reaches this path with used < packed_total. Before
|
||||
RAINFUSION_ATTN advertised supports_prefix_kv_slicing, the model built a
|
||||
padding mask here that _assert_metadata_compatible then rejected at the
|
||||
first sparse layer. This drives the real backend class (not a fake flag)
|
||||
through the packed path; both the sparse dispatch and the dense fallback
|
||||
share this metadata construction and neither ever reads attn_mask.
|
||||
"""
|
||||
from vllm_omni.diffusion.attention.backends.rainfusion_attn import (
|
||||
RainFusionAttentionBackend,
|
||||
)
|
||||
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
|
||||
MiniMaxH3Attention,
|
||||
)
|
||||
|
||||
class FakeAttention(torch.nn.Module):
|
||||
attn_backend = RainFusionAttentionBackend
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.metadata = None
|
||||
|
||||
def forward(self, query, key, value, metadata):
|
||||
self.metadata = metadata
|
||||
return query
|
||||
|
||||
attention = object.__new__(MiniMaxH3Attention)
|
||||
torch.nn.Module.__init__(attention)
|
||||
attention.attention = FakeAttention()
|
||||
q = torch.randn(8, 2, 4)
|
||||
|
||||
attention._run_packed_attention(
|
||||
q,
|
||||
q,
|
||||
q,
|
||||
# One request: 5 valid rows padded to the 8-row alignment boundary.
|
||||
cu_seqlens=torch.tensor([0, 5, 8], dtype=torch.int32),
|
||||
max_seqlen=5,
|
||||
packed_total=8,
|
||||
)
|
||||
|
||||
metadata = attention.attention.metadata
|
||||
assert metadata is not None
|
||||
assert metadata.attn_mask is None
|
||||
assert metadata.extra["valid_kv_length"] == 5
|
||||
|
||||
|
||||
def test_reference_image_resize_contract():
|
||||
from PIL import Image
|
||||
|
||||
@@ -1099,6 +1179,7 @@ def test_minimax_h3_advertises_the_official_ref2va_image_limit():
|
||||
from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata
|
||||
|
||||
assert get_diffusion_model_metadata("MiniMaxH3Pipeline").max_multimodal_image_inputs == 9
|
||||
assert get_diffusion_model_metadata("MiniMaxH3Pipeline").supports_latent_mask_editing
|
||||
|
||||
|
||||
def test_text_attention_routes_local_gqa_heads_through_sdpa_helper(monkeypatch):
|
||||
@@ -2756,41 +2837,63 @@ def test_dit_encoder_selection_keeps_vae_resident():
|
||||
component.offload_to_cpu.assert_not_called()
|
||||
|
||||
|
||||
def test_keyframe_encode_pins_and_restores_cudnn_settings():
|
||||
@pytest.mark.parametrize(
|
||||
("capability", "allow_tf32"),
|
||||
[((8, 6), True), ((9, 0), False), ((10, 0), True), ((10, 3), True), (None, True)],
|
||||
)
|
||||
@pytest.mark.parametrize("initial_tf32", [False, True])
|
||||
@pytest.mark.parametrize("fail_encode", [False, True])
|
||||
def test_keyframe_encode_pins_and_restores_cudnn_settings(
|
||||
monkeypatch, capability, allow_tf32, initial_tf32, fail_encode
|
||||
):
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.vae import (
|
||||
_minimax_h3_keyframe_encode_context,
|
||||
)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
def get_capability(device_id):
|
||||
assert device_id == 1
|
||||
return DeviceCapability(*capability) if capability is not None else None
|
||||
|
||||
monkeypatch.setattr(current_omni_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(current_omni_platform, "get_device_capability", get_capability)
|
||||
|
||||
cudnn = torch.backends.cudnn
|
||||
original = (
|
||||
cudnn.enabled,
|
||||
cudnn.benchmark,
|
||||
cudnn.deterministic,
|
||||
cudnn.allow_tf32,
|
||||
)
|
||||
try:
|
||||
cudnn.enabled = False
|
||||
cudnn.benchmark = True
|
||||
cudnn.deterministic = False
|
||||
cudnn.allow_tf32 = False
|
||||
|
||||
with _minimax_h3_keyframe_encode_context(torch.device("cuda")):
|
||||
with cudnn.flags(enabled=False, benchmark=True, deterministic=False, allow_tf32=initial_tf32):
|
||||
error_context = pytest.raises(RuntimeError, match="encode failed") if fail_encode else nullcontext()
|
||||
with error_context, _minimax_h3_keyframe_encode_context(torch.device("cuda:1")):
|
||||
assert cudnn.enabled
|
||||
assert not cudnn.benchmark
|
||||
assert cudnn.deterministic
|
||||
assert cudnn.allow_tf32
|
||||
assert cudnn.allow_tf32 is allow_tf32
|
||||
if fail_encode:
|
||||
raise RuntimeError("encode failed")
|
||||
|
||||
assert not cudnn.enabled
|
||||
assert cudnn.benchmark
|
||||
assert not cudnn.deterministic
|
||||
assert not cudnn.allow_tf32
|
||||
finally:
|
||||
(
|
||||
cudnn.enabled,
|
||||
cudnn.benchmark,
|
||||
cudnn.deterministic,
|
||||
cudnn.allow_tf32,
|
||||
) = original
|
||||
assert cudnn.allow_tf32 is initial_tf32
|
||||
|
||||
|
||||
def test_keyframe_encode_cpu_keeps_cudnn_settings(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3.vae import (
|
||||
_minimax_h3_keyframe_encode_context,
|
||||
)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
def unexpected_capability_query(device_id):
|
||||
pytest.fail("CPU keyframe encode must not query CUDA capability")
|
||||
|
||||
monkeypatch.setattr(current_omni_platform, "get_device_capability", unexpected_capability_query)
|
||||
cudnn = torch.backends.cudnn
|
||||
with cudnn.flags(enabled=False, benchmark=True, deterministic=False, allow_tf32=False):
|
||||
with _minimax_h3_keyframe_encode_context(torch.device("cpu")):
|
||||
assert not cudnn.enabled
|
||||
assert cudnn.benchmark
|
||||
assert not cudnn.deterministic
|
||||
assert not cudnn.allow_tf32
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail_encode", [False, True])
|
||||
@@ -2851,15 +2954,17 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
|
||||
class FakeEncoder:
|
||||
instances = []
|
||||
instances: list["FakeEncoder"] = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.pushes = []
|
||||
self.kwargs = kwargs
|
||||
self.__class__.instances.append(self)
|
||||
|
||||
def push(self, frames):
|
||||
def push(self, frames, *, on_consumed=None):
|
||||
self.pushes.append(np.array(frames, copy=True))
|
||||
if on_consumed is not None:
|
||||
on_consumed()
|
||||
|
||||
def finish(self):
|
||||
return b"mp4"
|
||||
@@ -2872,6 +2977,8 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
|
||||
return torch.zeros(1, 1, 2)
|
||||
|
||||
class FakeVideoVAE:
|
||||
chunk_value_range = (0.0, 1.0)
|
||||
|
||||
def decode_with_chunks(self, latent, *, on_chunk):
|
||||
for value in (0.0, 0.25, 0.5):
|
||||
on_chunk(torch.full((1, 3, 3, 2, 2), value))
|
||||
@@ -2883,7 +2990,7 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
|
||||
pipeline.device = torch.device("cpu")
|
||||
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.utils.media_utils.ChunkedMP4Encoder",
|
||||
"vllm_omni.diffusion.utils.chunked_video.ChunkedMP4Encoder",
|
||||
FakeEncoder,
|
||||
)
|
||||
|
||||
@@ -2903,19 +3010,90 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
|
||||
assert encoder.pushes[1].shape == (3, 2, 2, 3)
|
||||
|
||||
|
||||
def _peer_rank_preencode_pipeline(monkeypatch):
|
||||
"""An H3 pipeline whose video VAE is a distributed rank that owns no output.
|
||||
|
||||
Every rank of the VAE group must call ``decode_with_chunks`` to keep the
|
||||
temporal collectives in lockstep, but only the owner receives chunks, so a
|
||||
peer's callback is never invoked.
|
||||
"""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
|
||||
class FakeAudioVAE:
|
||||
def decode_latent(self, latent):
|
||||
return torch.zeros(1, 1, 2)
|
||||
|
||||
class PeerRankVideoVAE:
|
||||
chunk_value_range = (0.0, 1.0)
|
||||
|
||||
def decode_with_chunks(self, latent, *, on_chunk):
|
||||
del latent, on_chunk
|
||||
|
||||
pipeline = object.__new__(mod.MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.audio_vae = FakeAudioVAE()
|
||||
pipeline.video_vae = PeerRankVideoVAE()
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.od_config = SimpleNamespace()
|
||||
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_peer_vae_rank_returns_no_preencoded_output_instead_of_failing(monkeypatch):
|
||||
"""A rank that owns no output must not turn its silence into a request error."""
|
||||
pipeline = _peer_rank_preencode_pipeline(monkeypatch)
|
||||
|
||||
output = pipeline.decode_to_mp4(torch.zeros(1), torch.zeros(1), height=2, width=2)
|
||||
|
||||
assert output == b""
|
||||
|
||||
|
||||
def test_peer_vae_rank_post_decode_reaches_post_processing(monkeypatch):
|
||||
"""The real H3 post_decode chain, not just the shared consumer, tolerates a peer rank."""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
pipeline = _peer_rank_preencode_pipeline(monkeypatch)
|
||||
monkeypatch.setattr(pipeline, "_unpack_denoised_rows", lambda *args, **kwargs: (torch.zeros(1), torch.zeros(1)))
|
||||
state = StepRequestState(
|
||||
request_id="peer-rank",
|
||||
sampling=SimpleNamespace(num_outputs_per_prompt=1),
|
||||
prompt="a prompt",
|
||||
)
|
||||
state.latents = torch.zeros(1)
|
||||
state.extra[mod._STEP_BRANCH] = object()
|
||||
state.extra[mod._STEP_AUDIO_ROWS] = torch.zeros(1)
|
||||
state.extra[mod._STEP_SHAPE] = {
|
||||
"latent_t": 1,
|
||||
"latent_h": 1,
|
||||
"latent_w": 1,
|
||||
"audio_t": 1,
|
||||
"height": 2,
|
||||
"width": 2,
|
||||
"preencode_mp4": True,
|
||||
}
|
||||
|
||||
output = pipeline.post_decode(state)
|
||||
|
||||
assert output.output == (b"", None)
|
||||
assert mod._minimax_h3_post_process(output.output)["video"] == [b""]
|
||||
|
||||
|
||||
def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatch):
|
||||
"""A client's encoder options must survive the worker-side pre-encode path."""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
|
||||
class FakeEncoder:
|
||||
instances = []
|
||||
instances: list["FakeEncoder"] = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.__class__.instances.append(self)
|
||||
|
||||
def push(self, frames):
|
||||
def push(self, frames, *, on_consumed=None):
|
||||
del frames
|
||||
if on_consumed is not None:
|
||||
on_consumed()
|
||||
|
||||
def finish(self):
|
||||
return b"mp4"
|
||||
@@ -2928,6 +3106,8 @@ def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatc
|
||||
return torch.zeros(1, 1, 2)
|
||||
|
||||
class FakeVideoVAE:
|
||||
chunk_value_range = (0.0, 1.0)
|
||||
|
||||
def decode_with_chunks(self, latent, *, on_chunk):
|
||||
on_chunk(torch.zeros(1, 3, 1, 2, 2))
|
||||
|
||||
@@ -2937,7 +3117,7 @@ def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatc
|
||||
pipeline.video_vae = FakeVideoVAE()
|
||||
pipeline.device = torch.device("cpu")
|
||||
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
|
||||
monkeypatch.setattr("vllm_omni.diffusion.utils.media_utils.ChunkedMP4Encoder", FakeEncoder)
|
||||
monkeypatch.setattr("vllm_omni.diffusion.utils.chunked_video.ChunkedMP4Encoder", FakeEncoder)
|
||||
|
||||
pipeline.decode_to_mp4(
|
||||
torch.zeros(1),
|
||||
@@ -3010,3 +3190,19 @@ def test_video_codec_options_are_normalized_for_the_encoder():
|
||||
assert normalize_video_codec_options(None) is None
|
||||
with pytest.raises(ValueError, match="video_codec_options"):
|
||||
normalize_video_codec_options("preset=ultrafast")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("duration", [60, 75])
|
||||
def test_long_video_shape_requires_explicit_opt_in(duration):
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import resolve_minimax_h3_shape
|
||||
|
||||
sampling = OmniDiffusionSamplingParams(width=960, height=544, fps=24, extra_args={"duration": duration})
|
||||
with pytest.raises(OmniClientError, match="15"):
|
||||
resolve_minimax_h3_shape("ref2va", sampling, None)
|
||||
sampling.extra_args["long_video"] = True
|
||||
_, _, frames, video_t, audio_t = resolve_minimax_h3_shape("ref2va", sampling, None)
|
||||
assert frames >= duration * 24 and frames % 17 == 5
|
||||
assert video_t == (frames - 5) // 17 * 5 + 2
|
||||
assert audio_t == round(frames / 24 * 40)
|
||||
@@ -262,7 +262,10 @@ def test_a_vsa_variant_is_recognised_by_its_compression_gates(tmp_path):
|
||||
|
||||
def test_a_tensor_naming_no_h3_parameter_is_an_error(tmp_path):
|
||||
path = tmp_path / "fasth3" / "adapter_model.safetensors"
|
||||
_write_adapter(path, tensors={"transformer_blocks.0.attn.norm_q.lora_A.weight": torch.ones((_RANK, _HIDDEN))})
|
||||
_write_adapter(
|
||||
path,
|
||||
tensors={"transformer_blocks.0.attn.not_a_projection.lora_A.weight": torch.ones((_RANK, _HIDDEN))},
|
||||
)
|
||||
# Dropping it silently would load a model that is not the distilled student.
|
||||
with pytest.raises(FastH3AdapterError, match="name no known"):
|
||||
_load(path.parent)
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Pipeline-level MiniMax H3 latent initialization tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def test_prepare_edit_video_clones_the_last_frame_with_lossless_rgb(monkeypatch, tmp_path):
|
||||
from vllm_omni.model_executor.models.minimax_h3 import reference_video
|
||||
|
||||
source = tmp_path / "source.webm"
|
||||
commands: list[list[str]] = []
|
||||
monkeypatch.setattr(
|
||||
reference_video,
|
||||
"_probe_video",
|
||||
lambda _path: {"frame_count": 12, "audio_codecs": ("opus",)},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
reference_video.subprocess,
|
||||
"run",
|
||||
lambda command, **_kwargs: commands.append(command),
|
||||
)
|
||||
|
||||
prepared = reference_video.prepare_edit_video(
|
||||
source,
|
||||
target_width=448,
|
||||
target_height=256,
|
||||
target_frame_count=73,
|
||||
workdir=str(tmp_path / "prepared"),
|
||||
)
|
||||
|
||||
assert prepared["input_has_audio"] is True
|
||||
assert prepared["frame_count"] == 73
|
||||
command = commands[0]
|
||||
assert command[command.index("-vf") + 1].endswith("tpad=stop_mode=clone:stop=-1")
|
||||
assert command[command.index("-c:v") + 1] == "libx264rgb"
|
||||
assert command[command.index("-pix_fmt") + 1] == "rgb24"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source_audio", [None, "override.wav"])
|
||||
def test_encoder_preparation_routes_edit_sources_without_qwen_inputs(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
source_audio,
|
||||
):
|
||||
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing as processing
|
||||
|
||||
latent_t, audio_t = 7, 37
|
||||
video_clean = torch.full((42, 96), 3.0)
|
||||
audio_source = torch.zeros(4, 32)
|
||||
audio_source[:, 0] = torch.tensor([1.0, 2.0, 11.0, 12.0])
|
||||
waveform = torch.zeros(3_200)
|
||||
frames = torch.zeros(22, 64, 96, 3, dtype=torch.uint8).numpy()
|
||||
calls: dict[str, Any] = {}
|
||||
|
||||
def prepare_video(value, **kwargs):
|
||||
calls["prepare"] = (value, kwargs)
|
||||
return {
|
||||
"original_path": "source.mov",
|
||||
"prepared_path": str(tmp_path / "prepared.mp4"),
|
||||
"input_has_audio": True,
|
||||
}
|
||||
|
||||
def load_video_audio(value, **kwargs):
|
||||
calls["video_audio"] = (value, kwargs)
|
||||
return waveform, 32_000
|
||||
|
||||
def load_audio_file(value):
|
||||
calls["audio_file"] = value
|
||||
return waveform, 32_000
|
||||
|
||||
class VideoVAE:
|
||||
def encode_video(self, value):
|
||||
calls["video_encode_shape"] = value.shape
|
||||
return video_clean.clone(), (latent_t, 4, 6)
|
||||
|
||||
class AudioVAE:
|
||||
def encode_waveform(self, value, sample_rate):
|
||||
calls["audio_encode"] = (value.shape, sample_rate)
|
||||
return audio_source.clone(), 2
|
||||
|
||||
monkeypatch.setattr(processing, "resolve_minimax_h3_shape", lambda *_args: (64, 96, 22, latent_t, audio_t))
|
||||
monkeypatch.setattr(processing, "prepare_edit_video", prepare_video)
|
||||
monkeypatch.setattr(processing, "load_video_frames", lambda _path: frames)
|
||||
monkeypatch.setattr(processing, "load_video_audio", load_video_audio)
|
||||
monkeypatch.setattr(processing, "load_audio_file", load_audio_file)
|
||||
|
||||
prepared = processing.prepare_encoder_inputs(
|
||||
{
|
||||
"prompt": "edit the clip",
|
||||
"multi_modal_data": {
|
||||
"source_video": "source.mov",
|
||||
"source_audio": source_audio,
|
||||
"video_noise_mask": torch.zeros(1, 1, latent_t, 4, 6),
|
||||
"audio_noise_mask": torch.tensor([0.0, 0.5, *([0.0] * (audio_t - 2))]).reshape(1, 1, -1),
|
||||
},
|
||||
},
|
||||
SimpleNamespace(extra_args={"task": "t2va"}),
|
||||
)
|
||||
media = type(prepared.media).from_mm_tensors(
|
||||
prepared.media.to_mm_tensors(),
|
||||
prepared.media.to_metadata(),
|
||||
)
|
||||
conditioning = processing.encode_media(
|
||||
media,
|
||||
video_vae=VideoVAE(),
|
||||
audio_vae=AudioVAE(),
|
||||
emit_conditioning=True,
|
||||
)
|
||||
|
||||
assert prepared.images == []
|
||||
assert prepared.qwen_videos == []
|
||||
assert prepared.condition_labels == []
|
||||
assert prepared.media.images == ()
|
||||
assert prepared.media.videos == ()
|
||||
assert calls["prepare"][0] == "source.mov"
|
||||
assert calls["prepare"][1]["target_width"] == 96
|
||||
assert calls["prepare"][1]["target_height"] == 64
|
||||
assert calls["prepare"][1]["target_frame_count"] == 22
|
||||
if source_audio is None:
|
||||
assert calls["video_audio"] == (
|
||||
"source.mov",
|
||||
{"duration_seconds": pytest.approx(22 / 24)},
|
||||
)
|
||||
assert "audio_file" not in calls
|
||||
else:
|
||||
assert calls["audio_file"] == "override.wav"
|
||||
assert "video_audio" not in calls
|
||||
|
||||
assert conditioning is not None
|
||||
torch.testing.assert_close(conditioning.video_edit_clean_rows, video_clean)
|
||||
assert conditioning.video_edit_mask.shape == (latent_t, 4, 6)
|
||||
torch.testing.assert_close(conditioning.video_edit_mask, torch.zeros(latent_t, 4, 6))
|
||||
assert conditioning.audio_edit_source_t == 2
|
||||
assert conditioning.audio_edit_mask.shape == (2, audio_t)
|
||||
torch.testing.assert_close(
|
||||
conditioning.audio_edit_clean_rows[:, 0],
|
||||
torch.tensor([1.0, 2.0, *([0.0] * (audio_t - 2)), 11.0, 12.0, *([0.0] * (audio_t - 2))]),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
conditioning.audio_edit_mask,
|
||||
torch.tensor([0.0, 0.5, *([0.0] * (audio_t - 2))]).repeat(2, 1),
|
||||
)
|
||||
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
|
||||
MiniMaxH3EncoderConditioning,
|
||||
MiniMaxH3TextConditioning,
|
||||
)
|
||||
|
||||
combined = MiniMaxH3EncoderConditioning.from_components(
|
||||
MiniMaxH3TextConditioning(
|
||||
torch.zeros(1, 5120, dtype=torch.bfloat16),
|
||||
torch.ones(1, dtype=torch.int64),
|
||||
),
|
||||
conditioning,
|
||||
)
|
||||
round_trip = MiniMaxH3EncoderConditioning.from_omni_payload(combined.to_omni_payload())
|
||||
torch.testing.assert_close(round_trip.video_edit_clean_rows, video_clean)
|
||||
torch.testing.assert_close(round_trip.video_edit_mask, torch.zeros(latent_t, 4, 6))
|
||||
torch.testing.assert_close(round_trip.audio_edit_clean_rows, conditioning.audio_edit_clean_rows)
|
||||
torch.testing.assert_close(
|
||||
round_trip.audio_edit_mask,
|
||||
torch.tensor([0.0, 0.5, *([0.0] * (audio_t - 2))]).repeat(2, 1),
|
||||
)
|
||||
assert round_trip.audio_edit_source_t == 2
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.default_video_shift = 12.0
|
||||
pipeline.default_audio_shift = 3.0
|
||||
pipeline.od_config = SimpleNamespace(step_execution=False)
|
||||
pipeline._fasth3 = None
|
||||
pipeline._active_turbo_spec = lambda _sampling: None
|
||||
pipeline._has_active_native_lora = lambda _sampling: False
|
||||
pipeline._resolve_task = lambda task, **_kwargs: task
|
||||
pipeline._resolve_sigma_positions = lambda _task, _sampling: (None, 3)
|
||||
pipeline._quality_policy = SimpleNamespace(resolve=lambda **_kwargs: SimpleNamespace(cache_dit=None))
|
||||
pipeline._cache_dit_runtime = SimpleNamespace(prepare=lambda _plan: None)
|
||||
context = pipeline._prepare_encoder_conditioning_inputs(
|
||||
round_trip,
|
||||
SimpleNamespace(extra_args={}, quality=None, seed=17, num_outputs_per_prompt=1),
|
||||
)
|
||||
torch.testing.assert_close(context["video_edit_clean_rows"], video_clean)
|
||||
torch.testing.assert_close(context["video_edit_mask_rows"], torch.zeros(42))
|
||||
assert context["video_edit_restore_mask_rows"].shape == (42, 96)
|
||||
torch.testing.assert_close(
|
||||
context["audio_edit_mask_rows"],
|
||||
torch.tensor([0.0, 0.5, *([1.0] * (audio_t - 2))]).repeat(2),
|
||||
)
|
||||
torch.testing.assert_close(context["audio_edit_mask_rows"], context["audio_edit_restore_mask_rows"])
|
||||
|
||||
|
||||
def _build_inputs(*, video_mask: torch.Tensor, audio_mask: torch.Tensor):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
latent_t, latent_h, latent_w, audio_t = 2, 4, 6, 3
|
||||
video_clean = torch.linspace(-1.0, 1.0, latent_t * (latent_h // 2) * (latent_w // 2) * 96).reshape(-1, 96)
|
||||
audio_clean = torch.linspace(-0.5, 0.5, audio_t * 2 * 32).reshape(-1, 32)
|
||||
inputs = pipeline._build_denoise_inputs(
|
||||
task="t2va",
|
||||
text_embeddings=torch.zeros(2, 4),
|
||||
text_tags=torch.ones(2, dtype=torch.long),
|
||||
seed=17,
|
||||
latent_t=latent_t,
|
||||
latent_h=latent_h,
|
||||
latent_w=latent_w,
|
||||
audio_t=audio_t,
|
||||
num_frames=22,
|
||||
num_steps=3,
|
||||
video_shift=12.0,
|
||||
audio_shift=3.0,
|
||||
base_schedule=None,
|
||||
visual_condition=None,
|
||||
visual_condition_shape=None,
|
||||
audio_condition=None,
|
||||
ref_audio_t=None,
|
||||
video_edit_clean_rows=video_clean,
|
||||
video_edit_mask_rows=video_mask,
|
||||
video_edit_restore_mask_rows=video_mask,
|
||||
audio_edit_clean_rows=audio_clean,
|
||||
audio_edit_mask_rows=audio_mask,
|
||||
audio_edit_restore_mask_rows=audio_mask,
|
||||
)
|
||||
return pipeline, inputs, video_clean, audio_clean
|
||||
|
||||
|
||||
def test_build_denoise_inputs_uses_h3_source_anchors():
|
||||
pipeline, inputs, video_clean, audio_clean = _build_inputs(
|
||||
video_mask=torch.zeros(12),
|
||||
audio_mask=torch.zeros(6),
|
||||
)
|
||||
video_edit = inputs["video_edit"]
|
||||
audio_edit = inputs["audio_edit"]
|
||||
assert video_edit is not None and audio_edit is not None
|
||||
initial_video, _ = pipeline._initial_noise(
|
||||
seed=17,
|
||||
latent_t=2,
|
||||
latent_h=4,
|
||||
latent_w=6,
|
||||
audio_t=3,
|
||||
)
|
||||
torch.testing.assert_close(video_edit.clean_rows, video_clean)
|
||||
torch.testing.assert_close(video_edit.anchor_rows, 0.999 * video_clean + 0.001 * initial_video)
|
||||
torch.testing.assert_close(audio_edit.clean_rows, audio_clean)
|
||||
torch.testing.assert_close(audio_edit.anchor_rows, audio_clean)
|
||||
@@ -0,0 +1,81 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Focused CPU tests for MiniMax H3 latent-mask editing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def test_video_full_grid_quantization_and_cell_level_restore():
|
||||
from vllm_omni.diffusion.models.minimax_h3.latent_mask import minimax_h3_video_edit_masks
|
||||
|
||||
shape = dict(latent_t=2, latent_h=4, latent_w=6)
|
||||
full = torch.arange(48, dtype=torch.float32).reshape(2, 4, 6) / 48
|
||||
cells = full.reshape(2, 2, 2, 3, 2).permute(0, 1, 3, 2, 4)
|
||||
parsed = minimax_h3_video_edit_masks(full, **shape)
|
||||
expected_model = torch.ceil(cells.amax(dim=(3, 4)).reshape(-1) * 256) / 256
|
||||
expected_restore = cells.unsqueeze(3).expand(2, 2, 3, 24, 2, 2).reshape(12, 96)
|
||||
torch.testing.assert_close(parsed.model_mask_rows, expected_model)
|
||||
torch.testing.assert_close(parsed.restore_mask_rows, expected_restore)
|
||||
|
||||
|
||||
def test_audio_grid_uses_channel_major_order_and_preserves_raw_values():
|
||||
from vllm_omni.diffusion.models.minimax_h3.latent_mask import minimax_h3_audio_edit_masks
|
||||
|
||||
grid = torch.tensor([[0.0, 0.1, 1.0], [0.25, 0.5, 0.75]])
|
||||
expected = grid.flatten()
|
||||
parsed = minimax_h3_audio_edit_masks(grid, audio_t=3)
|
||||
torch.testing.assert_close(parsed.model_mask_rows, torch.ceil(expected * 256) / 256)
|
||||
torch.testing.assert_close(parsed.restore_mask_rows, expected)
|
||||
|
||||
|
||||
def test_zero_fractional_one_semantics_and_cell_level_restore():
|
||||
from vllm_omni.diffusion.models.minimax_h3.latent_mask import (
|
||||
MiniMaxH3LatentEdit,
|
||||
minimax_h3_video_edit_masks,
|
||||
)
|
||||
|
||||
clean = torch.full((3, 2), 10.0)
|
||||
anchor = torch.tensor([[8.0, 8.0], [6.0, 6.0], [4.0, 4.0]])
|
||||
state = torch.tensor([[2.0, 2.0], [4.0, 4.0], [6.0, 6.0]])
|
||||
velocity = torch.full((3, 2), 2.0)
|
||||
mask = torch.tensor([0.0, 0.5, 1.0])
|
||||
edit = MiniMaxH3LatentEdit.from_rows(clean, anchor, mask, mask)
|
||||
assert edit is not None
|
||||
|
||||
model_rows, target_timesteps = edit.prepare(state, 0.25, 0.999, sigma=0.75)
|
||||
torch.testing.assert_close(model_rows, torch.lerp(anchor, state, mask[:, None]))
|
||||
torch.testing.assert_close(
|
||||
target_timesteps,
|
||||
torch.tensor([0.999, 0.625, 0.25]),
|
||||
)
|
||||
predicted = model_rows + 0.75 * velocity * mask[:, None]
|
||||
torch.testing.assert_close(edit.x0(model_rows, velocity, 0.25), torch.lerp(clean, predicted, mask[:, None]))
|
||||
|
||||
# A single active cell drives the full model token, while raw restoration
|
||||
# preserves the other three cells independently in every packed channel.
|
||||
parsed = minimax_h3_video_edit_masks(
|
||||
torch.tensor([[[1.0, 0.0], [0.0, 0.0]]]),
|
||||
latent_t=1,
|
||||
latent_h=2,
|
||||
latent_w=2,
|
||||
)
|
||||
restore = torch.tensor([1.0, 0.0, 0.0, 0.0]).repeat(24).reshape(1, 96)
|
||||
torch.testing.assert_close(parsed.model_mask_rows, torch.ones(1))
|
||||
torch.testing.assert_close(parsed.restore_mask_rows, restore)
|
||||
|
||||
cell_edit = MiniMaxH3LatentEdit.from_rows(
|
||||
torch.full((1, 96), 10.0),
|
||||
torch.full((1, 96), 8.0),
|
||||
parsed.model_mask_rows,
|
||||
parsed.restore_mask_rows,
|
||||
)
|
||||
assert cell_edit is not None
|
||||
cell_state = torch.full((1, 96), 2.0)
|
||||
cell_velocity = torch.full((1, 96), 4.0)
|
||||
expected = torch.where(restore.bool(), cell_state + 0.75 * cell_velocity, cell_edit.clean_rows)
|
||||
torch.testing.assert_close(cell_edit.x0(cell_state, cell_velocity, 0.25), expected)
|
||||
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Duration limits apply before encoding and to transported encoder payloads."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import (
|
||||
prepare_encoder_inputs,
|
||||
resolve_minimax_h3_shape,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.long_video import validate_encoded_frame_limit
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode,limit", [("full", 30), ("continuation", 300), (None, 300)])
|
||||
@pytest.mark.parametrize("input_kind", ["duration", "duration_seconds", "target", "num_frames"])
|
||||
def test_long_video_limits_cover_all_duration_inputs(mode, limit, input_kind):
|
||||
extra: dict[str, Any] = {"long_video": True}
|
||||
if mode is not None:
|
||||
extra["long_video_mode"] = mode
|
||||
sampling = OmniDiffusionSamplingParams(width=64, height=64, fps=24, extra_args=extra)
|
||||
|
||||
def set_duration(seconds):
|
||||
if input_kind == "target":
|
||||
extra["target"] = {"duration_seconds": seconds}
|
||||
elif input_kind == "num_frames":
|
||||
sampling.num_frames = seconds * 24
|
||||
else:
|
||||
extra[input_kind] = seconds
|
||||
|
||||
set_duration(limit)
|
||||
_, _, frames, _, _ = resolve_minimax_h3_shape("ref2va", sampling, None)
|
||||
assert frames >= limit * 24
|
||||
validate_encoded_frame_limit(extra, "ref2va", frames)
|
||||
with pytest.raises(OmniClientError, match="request limit"):
|
||||
validate_encoded_frame_limit(extra, "ref2va", frames + 17)
|
||||
set_duration(limit + 1)
|
||||
with pytest.raises(OmniClientError, match="duration"):
|
||||
resolve_minimax_h3_shape("ref2va", sampling, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", ["t2va", "fl2va", "ref2va"])
|
||||
def test_full_mode_rejects_300_seconds_and_accepts_comparison(task):
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
width=64,
|
||||
height=64,
|
||||
fps=24,
|
||||
extra_args={"long_video": True, "long_video_mode": "full", "duration": 300, "aspect_ratio": "1:1"},
|
||||
)
|
||||
image = Image.new("RGB", (64, 64)) if task == "fl2va" else None
|
||||
with pytest.raises(OmniClientError, match="30"):
|
||||
resolve_minimax_h3_shape(task, sampling, image)
|
||||
sampling.extra_args["duration"] = 20
|
||||
assert resolve_minimax_h3_shape(task, sampling, image)[2] == 481
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["full", "continuation"])
|
||||
def test_mode_alone_does_not_bypass_long_video_opt_in(mode):
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
width=64,
|
||||
height=64,
|
||||
extra_args={"long_video_mode": mode, "duration": 20},
|
||||
)
|
||||
with pytest.raises(OmniClientError, match="15"):
|
||||
resolve_minimax_h3_shape("ref2va", sampling, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["video", "audio"])
|
||||
def test_continuation_edit_rejected_before_loading_source(kind):
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
width=64,
|
||||
height=64,
|
||||
extra_args={"long_video": True, "task": "ref2va", "duration": 20},
|
||||
)
|
||||
with pytest.raises(OmniClientError, match="continuation does not support latent-mask editing"):
|
||||
prepare_encoder_inputs(
|
||||
{
|
||||
"prompt": "test",
|
||||
"multi_modal_data": {
|
||||
"image": Image.new("RGB", (64, 64)),
|
||||
f"source_{kind}": "must-not-be-loaded",
|
||||
f"{kind}_noise_mask": 0.5,
|
||||
},
|
||||
},
|
||||
sampling,
|
||||
)
|
||||
|
||||
|
||||
def test_locked_audio_rejects_audio_edit_before_loading_source():
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
width=64,
|
||||
height=64,
|
||||
extra_args={"task": "t2va", "aspect_ratio": "1:1", "audio_mode": "lock_source"},
|
||||
)
|
||||
with pytest.raises(OmniClientError, match="cannot be combined"):
|
||||
prepare_encoder_inputs(
|
||||
{
|
||||
"prompt": "test",
|
||||
"multi_modal_data": {
|
||||
"audio": (torch.zeros(8000), 8000),
|
||||
"source_audio": "must-not-be-loaded",
|
||||
"audio_noise_mask": 0.5,
|
||||
},
|
||||
},
|
||||
sampling,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["video", "audio"])
|
||||
def test_external_encoder_cannot_bypass_continuation_edit_validation(kind):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(step_execution=False)
|
||||
conditioning = SimpleNamespace(
|
||||
task="ref2va",
|
||||
num_frames=481,
|
||||
video_edit_clean_rows=torch.zeros(1) if kind == "video" else None,
|
||||
audio_edit_clean_rows=torch.zeros(1) if kind == "audio" else None,
|
||||
)
|
||||
sampling = OmniDiffusionSamplingParams(extra_args={"long_video": True})
|
||||
with pytest.raises(OmniClientError, match="continuation does not support latent-mask editing"):
|
||||
pipeline._prepare_encoder_conditioning_inputs(conditioning, sampling)
|
||||
|
||||
|
||||
def test_external_encoder_cannot_bypass_full_duration_limit():
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(step_execution=False)
|
||||
sampling = OmniDiffusionSamplingParams(extra_args={"long_video": True, "long_video_mode": "full"})
|
||||
with pytest.raises(OmniClientError, match="30-second request limit"):
|
||||
pipeline._prepare_encoder_conditioning_inputs(SimpleNamespace(task="ref2va", num_frames=7213), sampling)
|
||||
@@ -190,3 +190,31 @@ def test_fused_modulation_preserves_bf16_residual_boundary() -> None:
|
||||
)
|
||||
|
||||
assert torch.equal(modulated_out, expected)
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="CUDA required")
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
|
||||
@pytest.mark.parametrize("fused_gate", [False, True])
|
||||
def test_fused_modulation_keeps_fp32_affine_intermediates(fused_gate: bool) -> None:
|
||||
device = current_omni_platform.get_torch_device()
|
||||
rows, hidden_size = 2, 5376
|
||||
x = torch.ones(rows, hidden_size, device=device, dtype=torch.bfloat16)
|
||||
weight = torch.full((hidden_size,), 1.125, device=device, dtype=x.dtype)
|
||||
shift = torch.zeros(1, hidden_size, device=device, dtype=x.dtype)
|
||||
scale = torch.full_like(shift, 0.09375)
|
||||
indices = torch.zeros(rows, device=device, dtype=torch.int64)
|
||||
eps = 1e-6
|
||||
if fused_gate:
|
||||
residual, actual = indexed_gate_rms_norm_scale_shift(
|
||||
x, torch.zeros_like(shift), x, weight, shift, scale, indices, eps
|
||||
)
|
||||
assert torch.equal(residual, x)
|
||||
else:
|
||||
actual = rms_norm_indexed_scale_shift(x, weight, shift, scale, indices, eps)
|
||||
|
||||
# Without the RMS epsilon, 1.125 * 1.09375 = 1.23046875 is exactly
|
||||
# halfway between two BF16 values. FP32 normalization keeps the epsilon
|
||||
# and rounds down to 1.2265625. Premature BF16 rounding drops the epsilon
|
||||
# and instead rounds the tie up to 1.234375, changing every output row.
|
||||
assert torch.equal(actual, torch.full_like(x, 1.2265625))
|
||||
@@ -86,6 +86,7 @@ def test_decode_releases_only_audio_before_return(monkeypatch):
|
||||
audio_vae=Mock(decode_latent=Mock(return_value=audio)),
|
||||
_component_on_device=lambda component: nullcontext(),
|
||||
_offload_model_cpu_stage_output=Mock(side_effect=lambda value: value),
|
||||
_release_stage_cache=Mock(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module.current_omni_platform,
|
||||
@@ -152,6 +153,7 @@ def test_forward_releases_video_after_quantization_and_before_next_seed(monkeypa
|
||||
diffuse=diffuse,
|
||||
decode=decode,
|
||||
_offload_model_cpu_stage_output=_recording_release(events, releases),
|
||||
_release_stage_cache=lambda: None,
|
||||
)
|
||||
request = SimpleNamespace(prompts=["a prompt"], sampling_params=SimpleNamespace())
|
||||
|
||||
@@ -201,6 +203,7 @@ def test_forward_drops_the_decoded_video_before_the_next_seed_diffuses(monkeypat
|
||||
diffuse=diffuse,
|
||||
decode=decode,
|
||||
_offload_model_cpu_stage_output=lambda value: value.clone(),
|
||||
_release_stage_cache=lambda: None,
|
||||
)
|
||||
request = SimpleNamespace(prompts=["a prompt"], sampling_params=SimpleNamespace())
|
||||
|
||||
@@ -231,6 +234,7 @@ def test_post_decode_releases_video_after_quantization(monkeypatch):
|
||||
_unpack_denoised_rows=lambda *args, **kwargs: (torch.zeros(1), torch.zeros(1)),
|
||||
decode=decode,
|
||||
_offload_model_cpu_stage_output=_recording_release(events, releases),
|
||||
_release_stage_cache=lambda: None,
|
||||
)
|
||||
state = SimpleNamespace(
|
||||
latents=torch.zeros(1),
|
||||
@@ -529,7 +533,7 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.audio_vae = Mock()
|
||||
pipeline._model_cpu_offload_modules = [pipeline.audio_vae]
|
||||
events = []
|
||||
events: list[tuple] = []
|
||||
|
||||
@contextmanager
|
||||
def record_component(value):
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
@@ -41,7 +42,7 @@ class _FakeAttention(nn.Module):
|
||||
super().__init__()
|
||||
|
||||
|
||||
def _small_od_config():
|
||||
def _small_od_config(*, class_name=None):
|
||||
arch = {
|
||||
"num_layers": 1,
|
||||
"token_refiner_num_layers": 1,
|
||||
@@ -60,6 +61,8 @@ def _small_od_config():
|
||||
"final_adaln_out_features": 2 * 8,
|
||||
"rope_inv_freq_len": 2,
|
||||
}
|
||||
if class_name is not None:
|
||||
arch["_class_name"] = class_name
|
||||
return SimpleNamespace(
|
||||
tf_model_config=arch,
|
||||
parallel_config=SimpleNamespace(ulysses_degree=1),
|
||||
@@ -133,6 +136,24 @@ def test_fp8_scope_and_prefix_propagation(monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
def test_explicit_native_weight_format_overrides_diffusers_class_name(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import minimax_h3_transformer as h3
|
||||
|
||||
monkeypatch.setattr(h3, "ColumnParallelLinear", _FakeLinear)
|
||||
monkeypatch.setattr(h3, "MergedColumnParallelLinear", _FakeLinear)
|
||||
monkeypatch.setattr(h3, "QKVParallelLinear", _FakeLinear)
|
||||
monkeypatch.setattr(h3, "RowParallelLinear", _FakeLinear)
|
||||
monkeypatch.setattr(h3, "Attention", _FakeAttention)
|
||||
monkeypatch.setattr(h3, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
|
||||
model = h3.MiniMaxH3DiTModel(
|
||||
_small_od_config(class_name="MiniMaxH3Transformer3DModel"),
|
||||
diffusers_weights=False,
|
||||
)
|
||||
|
||||
assert model._diffusers_weights is False
|
||||
|
||||
|
||||
class _WeightTarget(nn.Module):
|
||||
def __init__(self, loader):
|
||||
super().__init__()
|
||||
|
||||
@@ -187,8 +187,14 @@ def test_minimax_h3_quantization_quality(config: QualityTestConfig):
|
||||
"model": model_ref,
|
||||
"enforce_eager": True,
|
||||
# The fused BF16 baseline only fits on H100-80GB with encoder TP.
|
||||
"stage_0_tensor_parallel_size": 2,
|
||||
# This test loads the direct FL2VA diffusion pipeline (not the
|
||||
# disaggregated two-stage MiniMax-H3 pipeline), so configure both the
|
||||
# DiT world and the Qwen3-VL encoder TP explicitly.
|
||||
"tensor_parallel_size": 2,
|
||||
"text_encoder_tp_size": 2,
|
||||
"vae_use_tiling": True,
|
||||
# MiniMax H3's VAE is implemented by code shipped with the checkpoint.
|
||||
"trust_remote_code": True,
|
||||
}
|
||||
|
||||
omni_bl = Omni(**common_kwargs)
|
||||
|
||||
@@ -27,6 +27,9 @@ class _SegmentMeanModel:
|
||||
assignment shows up as a numeric difference instead of passing silently.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.calls: list[dict[str, torch.Tensor]] = []
|
||||
|
||||
def __call__(self, **kwargs):
|
||||
x = kwargs["x"][0]
|
||||
audio_x = kwargs["audio_x"][0]
|
||||
@@ -45,6 +48,14 @@ class _SegmentMeanModel:
|
||||
pooled_audio[start:stop] = audio_x[start:stop].mean(dim=0, keepdim=True)
|
||||
|
||||
row_timesteps = kwargs["unique_timesteps"][kwargs["inverse_indices"]].unsqueeze(-1)
|
||||
self.calls.append(
|
||||
{
|
||||
"video_rows": x[img_pos].clone(),
|
||||
"audio_rows": audio_x[audio_pos].clone(),
|
||||
"video_timesteps": row_timesteps[img_pos, 0].clone(),
|
||||
"audio_timesteps": row_timesteps[audio_pos, 0].clone(),
|
||||
}
|
||||
)
|
||||
video = (pooled_video + 0.5 * x + row_timesteps)[img_pos]
|
||||
audio = (pooled_audio + 0.5 * audio_x + row_timesteps)[audio_pos]
|
||||
if not kwargs.get("skip_mask_out_condition", False):
|
||||
@@ -156,9 +167,77 @@ def test_step_execution_matches_request_mode_denoise_loop():
|
||||
torch.testing.assert_close(state.extra[mod._STEP_AUDIO_ROWS], reference_audio)
|
||||
|
||||
|
||||
def test_batched_step_execution_matches_independent_requests():
|
||||
def test_step_execution_matches_request_mode_with_latent_edits():
|
||||
"""Masked model rows, row timesteps, and scheduler math match both paths."""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
from vllm_omni.diffusion.models.minimax_h3.denoise_loop import minimax_h3_denoise_loop
|
||||
from vllm_omni.diffusion.models.minimax_h3.latent_mask import MiniMaxH3LatentEdit
|
||||
|
||||
model = _SegmentMeanModel()
|
||||
branch, video_rows, audio_rows = _make_branch(text_len=9, latent_t=2, latent_h=4, latent_w=6, audio_t=3, seed=51)
|
||||
sigmas_video = _sigmas(5, 12.0)
|
||||
sigmas_audio = _sigmas(5, 3.0)
|
||||
video_clean = torch.linspace(-1.0, 1.0, video_rows.numel()).reshape_as(video_rows)
|
||||
audio_clean = torch.linspace(1.0, -1.0, audio_rows.numel()).reshape_as(audio_rows)
|
||||
video_mask = torch.linspace(0.0, 1.0, video_rows.shape[0])
|
||||
video_edit = MiniMaxH3LatentEdit.from_rows(
|
||||
video_clean,
|
||||
0.999 * video_clean + 0.001 * video_rows,
|
||||
video_mask,
|
||||
video_mask,
|
||||
)
|
||||
audio_mask = torch.tensor([0.0, 0.2, 0.4, 0.6, 0.8, 1.0])
|
||||
audio_edit = MiniMaxH3LatentEdit.from_rows(
|
||||
audio_clean,
|
||||
audio_clean,
|
||||
audio_mask,
|
||||
audio_mask,
|
||||
)
|
||||
|
||||
reference_video, reference_audio = minimax_h3_denoise_loop(
|
||||
model=model,
|
||||
positive=branch,
|
||||
initial_video_rows=video_rows,
|
||||
initial_audio_rows=audio_rows,
|
||||
keyframe_cond_rows=None,
|
||||
video_edit=video_edit,
|
||||
audio_edit=audio_edit,
|
||||
sigmas_video=sigmas_video,
|
||||
sigmas_audio=sigmas_audio,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
request_first_call = model.calls[0]
|
||||
torch.testing.assert_close(request_first_call["video_rows"], video_edit.model_rows(video_rows))
|
||||
torch.testing.assert_close(request_first_call["audio_rows"], audio_edit.model_rows(audio_rows))
|
||||
torch.testing.assert_close(
|
||||
request_first_call["video_timesteps"],
|
||||
video_edit.target_timesteps(1.0 - sigmas_video[0], 0.999, sigma=sigmas_video[0]),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
request_first_call["audio_timesteps"],
|
||||
audio_edit.target_timesteps(1.0 - sigmas_audio[0], 1.0, sigma=sigmas_audio[0]),
|
||||
)
|
||||
model.calls.clear()
|
||||
|
||||
state = _make_state("req-edit", model, branch, video_rows, audio_rows, sigmas_video, sigmas_audio)
|
||||
state.extra[mod._STEP_VIDEO_EDIT] = video_edit
|
||||
state.extra[mod._STEP_AUDIO_EDIT] = audio_edit
|
||||
pipeline = _step_pipeline(model)
|
||||
while not state.denoise_completed:
|
||||
velocity = pipeline.denoise_step(SimpleNamespace(states=(state,)), states=[state])
|
||||
pipeline.step_scheduler(state, velocity)
|
||||
|
||||
for name, expected in request_first_call.items():
|
||||
torch.testing.assert_close(model.calls[0][name], expected)
|
||||
torch.testing.assert_close(state.latents, reference_video)
|
||||
torch.testing.assert_close(state.extra[mod._STEP_AUDIO_ROWS], reference_audio)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lock_audio", [False, True])
|
||||
def test_batched_step_execution_matches_independent_requests(lock_audio):
|
||||
"""Two co-batched requests must land where they would have landed alone."""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
from vllm_omni.diffusion.models.minimax_h3.latent_mask import MiniMaxH3LatentEdit
|
||||
|
||||
model = _SegmentMeanModel()
|
||||
specs = [
|
||||
@@ -171,20 +250,40 @@ def test_batched_step_execution_matches_independent_requests():
|
||||
|
||||
pipeline = _step_pipeline(model)
|
||||
|
||||
def make_state(request_id: str, index: int):
|
||||
branch, video_rows, audio_rows = _make_branch(**specs[index])
|
||||
sigmas_video, sigmas_audio = schedules[index]
|
||||
state = _make_state(request_id, model, branch, video_rows, audio_rows, sigmas_video, sigmas_audio)
|
||||
video_mask = torch.linspace(0.1 * index, 0.8 + 0.1 * index, video_rows.shape[0])
|
||||
video_edit = MiniMaxH3LatentEdit.from_rows(
|
||||
torch.full_like(video_rows, 10.0 + index),
|
||||
torch.full_like(video_rows, 20.0 + index),
|
||||
video_mask,
|
||||
video_mask,
|
||||
)
|
||||
audio_mask = torch.linspace(0.2 - 0.1 * index, 1.0 - 0.1 * index, audio_rows.shape[0])
|
||||
audio_edit = MiniMaxH3LatentEdit.from_rows(
|
||||
torch.full_like(audio_rows, 30.0 + index),
|
||||
torch.full_like(audio_rows, 40.0 + index),
|
||||
audio_mask,
|
||||
audio_mask,
|
||||
)
|
||||
state.extra[mod._STEP_VIDEO_EDIT] = video_edit
|
||||
if lock_audio and index == 0:
|
||||
branch.locked_audio_rows = audio_rows.clone()
|
||||
else:
|
||||
state.extra[mod._STEP_AUDIO_EDIT] = audio_edit
|
||||
return state
|
||||
|
||||
alone: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
for spec, (sigmas_video, sigmas_audio) in zip(specs, schedules):
|
||||
branch, video_rows, audio_rows = _make_branch(**spec)
|
||||
state = _make_state("solo", model, branch, video_rows, audio_rows, sigmas_video, sigmas_audio)
|
||||
for index in range(len(specs)):
|
||||
state = make_state("solo", index)
|
||||
while not state.denoise_completed:
|
||||
pipeline.step_scheduler(state, pipeline.denoise_step(SimpleNamespace(states=(state,)), states=[state]))
|
||||
alone.append((state.latents, state.extra[mod._STEP_AUDIO_ROWS]))
|
||||
|
||||
states = []
|
||||
for index, (spec, (sigmas_video, sigmas_audio)) in enumerate(zip(specs, schedules)):
|
||||
branch, video_rows, audio_rows = _make_branch(**spec)
|
||||
if index == 0:
|
||||
assert branch.used_len == branch.seq_len
|
||||
states.append(_make_state(f"req-{index}", model, branch, video_rows, audio_rows, sigmas_video, sigmas_audio))
|
||||
states = [make_state(f"req-{index}", index) for index in range(len(specs))]
|
||||
assert states[0].extra[mod._STEP_BRANCH].used_len == states[0].extra[mod._STEP_BRANCH].seq_len
|
||||
|
||||
active = list(states)
|
||||
while active:
|
||||
@@ -291,6 +390,8 @@ def test_prepare_encode_seeds_runner_visible_state(monkeypatch, batch_frames):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
|
||||
branch, video_rows, audio_rows = _make_branch(text_len=9, latent_t=2, latent_h=4, latent_w=6, audio_t=3, seed=8)
|
||||
video_edit = object()
|
||||
audio_edit = object()
|
||||
sigmas_video = _sigmas(6, 12.0)
|
||||
sigmas_audio = _sigmas(6, 3.0)
|
||||
context = {
|
||||
@@ -326,6 +427,8 @@ def test_prepare_encode_seeds_runner_visible_state(monkeypatch, batch_frames):
|
||||
"audio_rows": audio_rows,
|
||||
"cond_anchor": None,
|
||||
"audio_anchor": None,
|
||||
"video_edit": video_edit,
|
||||
"audio_edit": audio_edit,
|
||||
"sigmas_video": sigmas_video,
|
||||
"sigmas_audio": sigmas_audio,
|
||||
},
|
||||
@@ -347,6 +450,8 @@ def test_prepare_encode_seeds_runner_visible_state(monkeypatch, batch_frames):
|
||||
assert state.do_true_cfg is False
|
||||
torch.testing.assert_close(state.current_timestep, torch.tensor(1.0 - sigmas_video[0]))
|
||||
assert state.extra[mod._STEP_BRANCH] is branch
|
||||
assert state.extra[mod._STEP_VIDEO_EDIT] is video_edit
|
||||
assert state.extra[mod._STEP_AUDIO_EDIT] is audio_edit
|
||||
assert state.extra[mod._STEP_SHAPE]["height"] == 96
|
||||
|
||||
pipeline.od_config = SimpleNamespace()
|
||||
@@ -450,3 +555,33 @@ def test_packed_batch_rejects_backends_that_cannot_isolate_requests(attention):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
assert MiniMaxH3Pipeline._packed_batch_supported(_FakeTransformer([attention])) is False
|
||||
|
||||
|
||||
def test_locked_driving_audio_is_clean_and_unchanged_during_denoising():
|
||||
from vllm_omni.diffusion.models.minimax_h3.denoise_loop import minimax_h3_denoise_loop
|
||||
|
||||
branch, video, audio = _make_branch(text_len=3, latent_t=2, latent_h=2, latent_w=2, audio_t=3, seed=7)
|
||||
branch.locked_audio_rows = audio.clone()
|
||||
seen = []
|
||||
|
||||
def model(**kwargs):
|
||||
positions = kwargs["audio_pos_info"]["position_ids"]
|
||||
times = kwargs["unique_timesteps"][kwargs["inverse_indices"]]
|
||||
torch.testing.assert_close(times[positions], torch.ones_like(times[positions]))
|
||||
torch.testing.assert_close(kwargs["audio_x"][0, positions], audio)
|
||||
seen.append(True)
|
||||
return torch.ones_like(video), torch.ones_like(audio)
|
||||
|
||||
result_video, result_audio = minimax_h3_denoise_loop(
|
||||
model=model,
|
||||
positive=branch,
|
||||
initial_video_rows=video,
|
||||
initial_audio_rows=audio,
|
||||
keyframe_cond_rows=None,
|
||||
sigmas_video=[1.0, 0.5, 0.0],
|
||||
sigmas_audio=[1.0, 0.25, 0.0],
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
assert len(seen) == 2
|
||||
torch.testing.assert_close(result_audio, audio)
|
||||
assert not torch.equal(result_video, video)
|
||||
@@ -0,0 +1,139 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""CPU tests for in-place decode revert and stage-boundary cache release.
|
||||
|
||||
``_revert_decoded_inplace`` must be numerically identical to the checkpoint's
|
||||
``revert_tensor`` (denormalize → clamp → rearrange) with zero whole-video
|
||||
copies, and ``_release_stage_cache`` must force the allocator-cache release
|
||||
(or fall back to a plain empty_cache when no component cache exists).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 as pipeline_module
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.diffusion.models.minimax_h3.vae import MiniMaxH3VideoVAE
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
DENORM_MEAN = (-2.11628412, -2.03571429, -1.80444444) # -imagenet_mean/std
|
||||
DENORM_STD = (4.36681223, 4.46428571, 4.44444444) # 1/imagenet_std
|
||||
|
||||
|
||||
class _FakeDenormalize:
|
||||
def __init__(self, mean, std):
|
||||
self.mean = mean
|
||||
self.std = std
|
||||
|
||||
|
||||
class _FakeProcessor:
|
||||
def __init__(self):
|
||||
self.transform_rev = _FakeDenormalize(DENORM_MEAN, DENORM_STD)
|
||||
self.use_3d_conv = True
|
||||
self.revert_calls = 0
|
||||
|
||||
def revert_tensor(self, tensor):
|
||||
self.revert_calls += 1
|
||||
return _legacy_revert(tensor)
|
||||
|
||||
|
||||
class _FakeCheckpointModel:
|
||||
def __init__(self):
|
||||
self.processor = _FakeProcessor()
|
||||
|
||||
|
||||
def _vae(model=None):
|
||||
vae = object.__new__(MiniMaxH3VideoVAE)
|
||||
vae.model = model if model is not None else _FakeCheckpointModel()
|
||||
return vae
|
||||
|
||||
|
||||
def _legacy_revert(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""The checkpoint's revert_tensor, op for op, on the CPU (B == 1)."""
|
||||
if tensor.ndim == 4:
|
||||
tensor = tensor.unsqueeze(2)
|
||||
assert tensor.ndim == 5
|
||||
# Normalize runs on the (b t) c h w rearrangement; at B == 1 that is a
|
||||
# view of (B, C, T, H, W), so the per-channel broadcast is equivalent.
|
||||
mean = torch.tensor(DENORM_MEAN).view(1, 3, 1, 1, 1)
|
||||
std = torch.tensor(DENORM_STD).view(1, 3, 1, 1, 1)
|
||||
tensor = (tensor - mean) / std
|
||||
return tensor.clamp(0, 1)
|
||||
|
||||
|
||||
def test_revert_inplace_matches_legacy_bitwise():
|
||||
decoded = torch.randn(1, 3, 7, 6, 8).clamp_(-3, 3)
|
||||
expected = _legacy_revert(decoded.clone())
|
||||
out = _vae()._revert_decoded_inplace(decoded.clone())
|
||||
assert out.shape == expected.shape == (1, 3, 7, 6, 8)
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
|
||||
def test_revert_inplace_mutates_without_extra_copies():
|
||||
decoded = torch.randn(1, 3, 5, 4, 4)
|
||||
out = _vae()._revert_decoded_inplace(decoded)
|
||||
assert out is decoded # in-place: no whole-video copy materializes
|
||||
assert out.is_contiguous()
|
||||
|
||||
|
||||
def test_revert_inplace_unsqueezes_4d_input():
|
||||
# 4-D decoded tensors follow the (B, C, H, W) image contract: unsqueeze(2)
|
||||
# lands T=1, matching the checkpoint's revert_tensor branch.
|
||||
decoded = torch.randn(1, 3, 6, 8)
|
||||
expected = _legacy_revert(decoded.clone())
|
||||
out = _vae()._revert_decoded_inplace(decoded.clone())
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
|
||||
def test_revert_inplace_falls_back_without_contract():
|
||||
vae = _vae()
|
||||
vae.model.processor.transform_rev = type("Empty", (), {})()
|
||||
decoded = torch.randn(1, 3, 5, 4, 4)
|
||||
out = vae._revert_decoded_inplace(decoded)
|
||||
assert vae.model.processor.revert_calls == 1
|
||||
assert torch.equal(out, _legacy_revert(decoded))
|
||||
|
||||
|
||||
def test_revert_inplace_falls_back_on_bad_arity():
|
||||
vae = _vae()
|
||||
vae.model.processor.transform_rev = _FakeDenormalize((0.5, 0.5), (1.0, 1.0, 1.0))
|
||||
vae._revert_decoded_inplace(torch.randn(1, 3, 5, 4, 4))
|
||||
assert vae.model.processor.revert_calls == 1
|
||||
|
||||
|
||||
class _RecordingCache:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def release_if_needed(self, *, force=False):
|
||||
self.calls.append(force)
|
||||
return True
|
||||
|
||||
|
||||
def _pipeline():
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
pipeline._dlo_component_cache = None
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_stage_cache_release_forces_component_cache():
|
||||
pipeline = _pipeline()
|
||||
cache = _RecordingCache()
|
||||
pipeline._dlo_component_cache = cache
|
||||
pipeline._release_stage_cache()
|
||||
assert cache.calls == [True]
|
||||
|
||||
|
||||
def test_stage_cache_release_falls_back_to_empty_cache(monkeypatch):
|
||||
released = []
|
||||
monkeypatch.setattr(
|
||||
pipeline_module.current_omni_platform,
|
||||
"empty_cache",
|
||||
lambda: released.append(True),
|
||||
raising=True,
|
||||
)
|
||||
pipeline = _pipeline()
|
||||
pipeline._release_stage_cache()
|
||||
assert released == [True]
|
||||
@@ -0,0 +1,243 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""CPU tests for the streaming uint8 preparation in the MiniMax-H3 video VAE.
|
||||
|
||||
``_stream_prepare_video_tensor`` must be numerically identical to the
|
||||
checkpoint's whole-video numpy path (``convert_numpy_to_tensor`` →
|
||||
``transform_tensor`` → ``transpose``) while padding on the host, so the
|
||||
device-side copies never materialize.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.vae import MiniMaxH3VideoVAE
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
||||
IMAGENET_STD = (0.229, 0.224, 0.225)
|
||||
CLIP_LENGTH = 4
|
||||
|
||||
|
||||
class _FakeNormalize:
|
||||
def __init__(self, mean, std):
|
||||
self.mean = mean
|
||||
self.std = std
|
||||
|
||||
|
||||
class _FakeProcessor:
|
||||
# Read by the stream prep via getattr(..., False); annotation only so
|
||||
# tests can attach it per case.
|
||||
isolated_last_frame: bool
|
||||
|
||||
def __init__(self):
|
||||
self.transform = _FakeNormalize(IMAGENET_MEAN, IMAGENET_STD)
|
||||
|
||||
|
||||
class _FakeCheckpointModel:
|
||||
def __init__(self, **overrides):
|
||||
self.clip_length = CLIP_LENGTH
|
||||
self.isolated_first_frame = False
|
||||
self.frame_pre_padding = 0
|
||||
self.processor = _FakeProcessor()
|
||||
for key, value in overrides.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
|
||||
def _vae(model=None):
|
||||
vae = object.__new__(MiniMaxH3VideoVAE)
|
||||
vae.model = model if model is not None else _FakeCheckpointModel()
|
||||
return vae
|
||||
|
||||
|
||||
def _legacy_reference(frames: np.ndarray) -> torch.Tensor:
|
||||
"""The checkpoint's numpy path, op for op, on the CPU.
|
||||
|
||||
convert_numpy_to_tensor: astype(fp32) → permute → ÷255
|
||||
transform_tensor: Normalize = (x - mean) / std
|
||||
encode_videos: transpose(0, 1)
|
||||
"""
|
||||
tensor = torch.from_numpy(frames.astype(np.float32)).permute(0, 3, 1, 2)
|
||||
tensor = tensor / 255.0
|
||||
mean = torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1)
|
||||
std = torch.tensor(IMAGENET_STD).view(1, 3, 1, 1)
|
||||
tensor = (tensor - mean) / std
|
||||
return tensor.transpose(0, 1).contiguous()
|
||||
|
||||
|
||||
def _video(num_frames, height=6, width=8, seed=0):
|
||||
rng = np.random.default_rng(seed)
|
||||
return rng.integers(0, 256, size=(num_frames, height, width, 3), dtype=np.uint8)
|
||||
|
||||
|
||||
def _pad_frames(frames: np.ndarray, pad: int) -> np.ndarray:
|
||||
if not pad:
|
||||
return frames
|
||||
return np.concatenate([frames, np.repeat(frames[-1:], pad, axis=0)])
|
||||
|
||||
|
||||
def test_stream_prep_matches_legacy_path_bitwise():
|
||||
frames = _video(10)
|
||||
out = _vae()._stream_prepare_video_tensor(frames, torch.device("cpu"))
|
||||
assert out is not None
|
||||
assert out.shape == (3, 12, 6, 8) # pad = (0 - 10) % 4 == 2
|
||||
assert out.dtype == torch.float32
|
||||
assert torch.equal(out, _legacy_reference(_pad_frames(frames, 2)))
|
||||
|
||||
|
||||
def test_stream_prep_isolated_first_frame_without_last_frame_skips_pad():
|
||||
# Asymmetric checkpoint (isolated first frame, no isolated last frame):
|
||||
# trim-stable lengths are k*clip + 0, which never satisfy the
|
||||
# encode_temporal offset of 1, so the remote re-pads through its own
|
||||
# whole-video cat regardless. Padding here would only upload frames the
|
||||
# remote trim discards, so the legacy frame count is kept.
|
||||
model = _FakeCheckpointModel(isolated_first_frame=True)
|
||||
frames = _video(10)
|
||||
out = _vae(model)._stream_prepare_video_tensor(frames, torch.device("cpu"))
|
||||
assert out is not None
|
||||
assert out.shape[1] == 10
|
||||
assert torch.equal(out, _legacy_reference(frames))
|
||||
|
||||
|
||||
def test_stream_prep_symmetric_isolated_frames_pad_to_trim_stable():
|
||||
# isolated first + last frames: offset == tail % clip == 1, so padding
|
||||
# to the next trim-stable length makes the remote trim a no-op and keeps
|
||||
# encode_temporal from re-padding: T=10, clip=4, tail=1 → 3*4+1 == 13.
|
||||
model = _FakeCheckpointModel(isolated_first_frame=True)
|
||||
model.processor.isolated_last_frame = True
|
||||
frames = _video(10)
|
||||
out = _vae(model)._stream_prepare_video_tensor(frames, torch.device("cpu"))
|
||||
assert out is not None
|
||||
assert out.shape[1] == 13
|
||||
assert out.shape[1] % 4 == 1
|
||||
assert torch.equal(out, _legacy_reference(_pad_frames(frames, 3)))
|
||||
|
||||
|
||||
def test_stream_prep_uses_processor_align_when_available():
|
||||
calls = []
|
||||
|
||||
class _AlignedProcessor(_FakeProcessor):
|
||||
def align_video_length(self, video_length, mode="pad", granularity="chunk"):
|
||||
calls.append((video_length, mode, granularity))
|
||||
return 5
|
||||
|
||||
model = _FakeCheckpointModel()
|
||||
model.processor = _AlignedProcessor()
|
||||
frames = _video(10)
|
||||
out = _vae(model)._stream_prepare_video_tensor(frames, torch.device("cpu"))
|
||||
assert out is not None
|
||||
assert calls == [(10, "pad", "chunk")]
|
||||
assert out.shape[1] == 15
|
||||
assert torch.equal(out, _legacy_reference(_pad_frames(frames, 5)))
|
||||
|
||||
|
||||
def test_stream_prep_pads_with_last_frame():
|
||||
frames = _video(2) # pad = (0 - 2) % 4 == 2
|
||||
out = _vae()._stream_prepare_video_tensor(frames, torch.device("cpu"))
|
||||
padded = np.concatenate([frames, np.repeat(frames[-1:], 2, axis=0)])
|
||||
assert torch.equal(out, _legacy_reference(padded))
|
||||
|
||||
|
||||
def test_stream_prep_aligned_input_needs_no_pad():
|
||||
frames = _video(8) # 8 % 4 == 0
|
||||
out = _vae()._stream_prepare_video_tensor(frames, torch.device("cpu"))
|
||||
assert out.shape[1] == 8
|
||||
assert torch.equal(out, _legacy_reference(frames))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"frames",
|
||||
[
|
||||
_video(5).astype(np.float32), # not uint8
|
||||
_video(5)[:, :, :, :1], # single channel
|
||||
np.zeros((5, 6, 8, 4), dtype=np.uint8), # four channels
|
||||
np.zeros((5, 6, 8), dtype=np.uint8), # missing channel axis
|
||||
np.zeros((0, 6, 8, 3), dtype=np.uint8), # empty
|
||||
],
|
||||
)
|
||||
def test_stream_prep_rejects_unsupported_inputs(frames):
|
||||
assert _vae()._stream_prepare_video_tensor(frames, torch.device("cpu")) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
_FakeCheckpointModel(clip_length=None),
|
||||
_FakeCheckpointModel(clip_length=0),
|
||||
type("NoProcessor", (), {"clip_length": CLIP_LENGTH})(),
|
||||
type(
|
||||
"NoTransform",
|
||||
(),
|
||||
{"clip_length": CLIP_LENGTH, "processor": type("P", (), {})()},
|
||||
)(),
|
||||
],
|
||||
)
|
||||
def test_stream_prep_rejects_missing_checkpoint_contract(model):
|
||||
frames = _video(5)
|
||||
assert _vae(model)._stream_prepare_video_tensor(frames, torch.device("cpu")) is None
|
||||
|
||||
|
||||
def test_stream_prep_rejects_bad_norm_arity():
|
||||
model = _FakeCheckpointModel()
|
||||
model.processor.transform = _FakeNormalize((0.5, 0.5), IMAGENET_STD)
|
||||
assert _vae(model)._stream_prepare_video_tensor(_video(5), torch.device("cpu")) is None
|
||||
|
||||
|
||||
class _RecordingModel(_FakeCheckpointModel):
|
||||
"""Feeds encode_video a well-shaped latent and records its input."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.received = None
|
||||
self.latent = torch.randn(1, 4, 3, 4, 4) # (B, C, T', H', W'), even H/W
|
||||
|
||||
def encode_videos(self, videos, use_fp16_latent=False, **kwargs):
|
||||
self.received = videos
|
||||
assert use_fp16_latent is True
|
||||
return [self.latent]
|
||||
|
||||
|
||||
def _vae_for_encode_video():
|
||||
vae = object.__new__(MiniMaxH3VideoVAE)
|
||||
vae.model = _RecordingModel()
|
||||
vae.device_module = torch
|
||||
vae.config_dict = {
|
||||
"latent_channels": 4,
|
||||
"latents_mean": [0.0] * 4,
|
||||
"latents_std": [1.0] * 4,
|
||||
}
|
||||
# A plain tensor (not nn.Parameter): nn.Module.__setattr__ refuses
|
||||
# Parameters on instances built via object.__new__.
|
||||
vae._dummy_parameter = torch.zeros(1, dtype=torch.float32)
|
||||
vae.parameters = lambda: iter([vae._dummy_parameter])
|
||||
return vae
|
||||
|
||||
|
||||
def test_encode_video_streams_uint8_numpy(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_OMNI_VAE_ENCODE_LEGACY_PREP", raising=False)
|
||||
vae = _vae_for_encode_video()
|
||||
rows, shape = vae.encode_video(_video(10))
|
||||
assert isinstance(vae.model.received, list)
|
||||
assert isinstance(vae.model.received[0], torch.Tensor)
|
||||
assert vae.model.received[0].shape == (3, 12, 6, 8)
|
||||
assert rows.shape[0] == shape[0] * shape[1] * shape[2] // 4 # patchify rows
|
||||
|
||||
|
||||
def test_encode_video_legacy_env_escapes_to_checkpoint_path(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_OMNI_VAE_ENCODE_LEGACY_PREP", "1")
|
||||
vae = _vae_for_encode_video()
|
||||
frames = _video(10)
|
||||
vae.encode_video(frames)
|
||||
assert vae.model.received is frames # untouched numpy, checkpoint converts
|
||||
|
||||
|
||||
def test_encode_video_falls_back_when_preparation_unsupported(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_OMNI_VAE_ENCODE_LEGACY_PREP", raising=False)
|
||||
vae = _vae_for_encode_video()
|
||||
vae.model.clip_length = None # contract not discoverable
|
||||
frames = _video(10)
|
||||
vae.encode_video(frames)
|
||||
assert vae.model.received is frames
|
||||
@@ -0,0 +1,218 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""CPU tests for the split encoder/decoder residency of the MiniMax-H3 VAE.
|
||||
|
||||
The encode half (CNN encoder + quant_conv) and the decode half
|
||||
(post_quant_conv + ViT decoder) stage independently: loading one half must
|
||||
rebind only its own parameters while the other half stays on its pinned
|
||||
CPU master.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.vae import (
|
||||
MiniMaxH3VideoVAE,
|
||||
_VideoVAEPartProxy,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class _FakeRemoteModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.encoder = nn.Linear(8, 8)
|
||||
self.quant_conv = nn.Linear(4, 4)
|
||||
self.post_quant_conv = nn.Linear(4, 4)
|
||||
self.decoder = nn.Linear(16, 16)
|
||||
|
||||
|
||||
class _FakeRemote(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.model = _FakeRemoteModel()
|
||||
|
||||
|
||||
def _vae() -> MiniMaxH3VideoVAE:
|
||||
vae = object.__new__(MiniMaxH3VideoVAE)
|
||||
nn.Module.__init__(vae) # bare instance: enable Module attribute assignment
|
||||
vae._device_target = torch.device("cpu")
|
||||
vae.remote = _FakeRemote()
|
||||
vae.model = vae.remote.model
|
||||
vae._stager = None
|
||||
vae._encoder_stager = None
|
||||
vae._decoder_stager = None
|
||||
vae._build_residency_stagers(torch.device("cpu"))
|
||||
vae.encoder_component = _VideoVAEPartProxy(vae, "encoder")
|
||||
vae.decoder_component = _VideoVAEPartProxy(vae, "decoder")
|
||||
return vae
|
||||
|
||||
|
||||
def _ptr(param: torch.nn.Parameter) -> int:
|
||||
return param.untyped_storage().data_ptr()
|
||||
|
||||
|
||||
def test_split_stagers_snapshot_disjoint_halves():
|
||||
vae = _vae()
|
||||
enc_master = _ptr(vae.model.encoder.weight)
|
||||
dec_master = _ptr(vae.model.decoder.weight)
|
||||
pqc_master = _ptr(vae.model.post_quant_conv.weight)
|
||||
assert enc_master != dec_master
|
||||
|
||||
vae._load_part_to_device("encoder")
|
||||
# The encoder half is rebound to freshly allocated staging storage...
|
||||
assert _ptr(vae.model.encoder.weight) != enc_master
|
||||
assert _ptr(vae.model.quant_conv.weight) != enc_master
|
||||
# ...while every decode-half tensor stays on its own CPU master.
|
||||
assert _ptr(vae.model.decoder.weight) == dec_master
|
||||
assert _ptr(vae.model.post_quant_conv.weight) == pqc_master
|
||||
|
||||
vae._offload_part_to_cpu("encoder")
|
||||
assert _ptr(vae.model.encoder.weight) == enc_master
|
||||
|
||||
|
||||
def test_part_load_is_idempotent_and_independent():
|
||||
vae = _vae()
|
||||
vae._load_part_to_device("decoder")
|
||||
staged_ptr = _ptr(vae.model.decoder.weight)
|
||||
vae._load_part_to_device("decoder")
|
||||
assert _ptr(vae.model.decoder.weight) == staged_ptr
|
||||
vae._offload_part_to_cpu("decoder")
|
||||
|
||||
|
||||
def test_whole_component_load_and_offload_cover_both_halves():
|
||||
vae = _vae()
|
||||
enc_master = _ptr(vae.model.encoder.weight)
|
||||
dec_master = _ptr(vae.model.decoder.weight)
|
||||
vae.load_to_device()
|
||||
assert _ptr(vae.model.encoder.weight) != enc_master
|
||||
assert _ptr(vae.model.decoder.weight) != dec_master
|
||||
vae.offload_to_cpu()
|
||||
assert _ptr(vae.model.encoder.weight) == enc_master
|
||||
assert _ptr(vae.model.decoder.weight) == dec_master
|
||||
|
||||
|
||||
def test_part_values_survive_a_load_offload_cycle():
|
||||
vae = _vae()
|
||||
expected = vae.model.encoder.weight.detach().clone()
|
||||
vae._load_part_to_device("encoder")
|
||||
assert torch.equal(vae.model.encoder.weight.detach(), expected)
|
||||
vae._offload_part_to_cpu("encoder")
|
||||
assert torch.equal(vae.model.encoder.weight.detach(), expected)
|
||||
|
||||
|
||||
def test_proxy_routes_to_the_matching_half():
|
||||
vae = _vae()
|
||||
enc_master = _ptr(vae.model.encoder.weight)
|
||||
dec_master = _ptr(vae.model.decoder.weight)
|
||||
vae.encoder_component.load_to_device()
|
||||
assert _ptr(vae.model.encoder.weight) != enc_master
|
||||
assert _ptr(vae.model.decoder.weight) == dec_master
|
||||
vae.encoder_component.offload_to_cpu()
|
||||
assert _ptr(vae.model.encoder.weight) == enc_master
|
||||
|
||||
|
||||
def test_proxy_does_not_register_the_adapter_as_submodule():
|
||||
vae = _vae()
|
||||
proxy = vae.encoder_component
|
||||
assert proxy._vae is vae
|
||||
# The back-reference must not recurse through module traversal.
|
||||
assert all(child is not vae for child in proxy.children())
|
||||
assert len(list(proxy.parameters())) == 0
|
||||
|
||||
|
||||
def test_component_on_device_unwraps_proxy_for_sequential_offload(monkeypatch):
|
||||
"""Model-level CPU offload hooks the real video_vae, never the part
|
||||
proxies: _component_on_device must hand the hooked module to the
|
||||
sequential context, or _get_sequential_offload_hook raises on the
|
||||
hook-less proxy on the first encode/decode of every request."""
|
||||
from contextlib import contextmanager
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as pipeline_module
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
vae = _vae()
|
||||
seen: list[nn.Module] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_sequential_offload_component(component):
|
||||
seen.append(component)
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "sequential_offload_component", fake_sequential_offload_component)
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline._model_cpu_offload_modules = [vae]
|
||||
|
||||
with pipeline._component_on_device(vae.encoder_component):
|
||||
pass
|
||||
with pipeline._component_on_device(vae.decoder_component):
|
||||
pass
|
||||
# A plain component passes through unchanged.
|
||||
with pipeline._component_on_device(vae):
|
||||
pass
|
||||
# Mocks auto-create any attribute, so the unwrap must be a type check:
|
||||
# a Mock component must reach the sequential context as itself.
|
||||
mock_vae = Mock()
|
||||
pipeline._model_cpu_offload_modules = [mock_vae]
|
||||
with pipeline._component_on_device(mock_vae):
|
||||
pass
|
||||
|
||||
assert seen == [vae, vae, vae, mock_vae]
|
||||
|
||||
|
||||
def test_sequential_offload_target_is_the_hooked_module():
|
||||
from vllm_omni.diffusion.offloader.sequential_backend import _get_sequential_offload_hook
|
||||
|
||||
vae = _vae()
|
||||
proxy = vae.encoder_component
|
||||
# The pre-fix failure mode: the proxy carries no hook registry.
|
||||
with pytest.raises(RuntimeError, match="no sequential offload hook"):
|
||||
_get_sequential_offload_hook(proxy)
|
||||
assert proxy.sequential_offload_target is vae
|
||||
|
||||
|
||||
def test_cache_retention_forwarded_to_both_stagers():
|
||||
vae = _vae()
|
||||
cache = object()
|
||||
vae.set_omni_component_cache(cache)
|
||||
assert vae._encoder_stager.cache_retention is cache
|
||||
assert vae._decoder_stager.cache_retention is cache
|
||||
|
||||
|
||||
def test_unknown_part_raises():
|
||||
vae = _vae()
|
||||
with pytest.raises(ValueError, match="unknown video VAE part"):
|
||||
vae._part_stager("middle")
|
||||
|
||||
|
||||
def test_missing_checkpoint_structure_falls_back_to_whole_stager():
|
||||
vae = object.__new__(MiniMaxH3VideoVAE)
|
||||
nn.Module.__init__(vae)
|
||||
vae._device_target = torch.device("cpu")
|
||||
|
||||
class BareRemote(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.model = nn.Linear(4, 4) # no encoder/decoder submodules
|
||||
|
||||
vae.remote = BareRemote()
|
||||
vae.model = vae.remote.model
|
||||
vae._stager = None
|
||||
vae._encoder_stager = None
|
||||
vae._decoder_stager = None
|
||||
vae._build_residency_stagers(torch.device("cpu"))
|
||||
assert vae._stager is not None
|
||||
assert vae._encoder_stager is None
|
||||
assert vae._decoder_stager is None
|
||||
# Part loads fall back to the whole-module stager.
|
||||
vae._load_part_to_device("decoder")
|
||||
assert vae._stager.loaded
|
||||
vae._offload_part_to_cpu("decoder")
|
||||
assert not vae._stager.loaded
|
||||
@@ -0,0 +1,335 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""CPU tests for the temporal VAE forks and the uint8 output transfer.
|
||||
|
||||
The encode fork must feed bit-identical frames to the checkpoint's chunk
|
||||
loop (padding moves from a whole-video cat into the tail chunk), the decode
|
||||
fork must produce the same video as the legacy fp32-buffer chain
|
||||
(revert -> clamp -> *255 -> round -> uint8), and the output transfer must
|
||||
match the previous ``.to(uint8, contiguous)`` conversion.
|
||||
"""
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_prepare_minimax_h3_video_output,
|
||||
)
|
||||
from vllm_omni.diffusion.models.minimax_h3.vae_temporal import (
|
||||
_decode_temporal_streaming_uint8,
|
||||
_encode_temporal_tail_pad,
|
||||
install_temporal_stream_patches,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
CLIP = 4
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# encode_temporal fork
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _EncodeModel:
|
||||
"""Minimal stand-in implementing the checkpoint's encode_temporal."""
|
||||
|
||||
# Attached per test case as capability probes for the install checks;
|
||||
# annotations only, the fork must keep working when they are absent.
|
||||
encode_temporal: Any
|
||||
_decode_temporal_streaming: Any
|
||||
processor: Any
|
||||
|
||||
def __init__(self, *, isolated_first=False, isolated_last=False, key_frame=False, frame_drop=0):
|
||||
self.clip_length = CLIP
|
||||
self.isolated_first_frame = isolated_first
|
||||
self.frame_pre_padding = 0
|
||||
self.isolated_key_frame = key_frame
|
||||
self.token_drop = 0
|
||||
self.isolated_last_frame = isolated_last
|
||||
self.frame_drop = frame_drop
|
||||
self.encoded = []
|
||||
|
||||
def _adaptive_encode(self, clip):
|
||||
self.encoded.append(clip.clone())
|
||||
return clip # identity: z_list entries are the clips themselves
|
||||
|
||||
def reference_encode_temporal(self, x):
|
||||
"""The checkpoint's original: whole-video pad, then chunk."""
|
||||
offset_frame = 1 if self.isolated_first_frame and self.frame_pre_padding == 0 else 0
|
||||
if x.shape[2] % self.clip_length != offset_frame:
|
||||
pad_size = (offset_frame - x.shape[2]) % self.clip_length
|
||||
pad_frames = x[:, :, -1:].repeat(1, 1, pad_size, 1, 1)
|
||||
x = torch.cat([x, pad_frames], dim=2)
|
||||
num_chunks = (x.shape[2] - offset_frame) // self.clip_length
|
||||
z_list = []
|
||||
for i in range(num_chunks):
|
||||
start_idx = i * self.clip_length + offset_frame
|
||||
end_idx = (i + 1) * self.clip_length + offset_frame
|
||||
clip_x = x[:, :, start_idx:end_idx, :, :]
|
||||
if self.isolated_key_frame:
|
||||
key_frame = clip_x[:, :, :1, :, :]
|
||||
z_key = self._adaptive_encode(key_frame)
|
||||
if clip_x.shape[2] > 1:
|
||||
z_video = self._adaptive_encode(clip_x[:, :, 1:, :, :])
|
||||
z = torch.cat([z_key, z_video], dim=2)
|
||||
else:
|
||||
z = z_key
|
||||
else:
|
||||
z = self._adaptive_encode(clip_x)
|
||||
z_list.append(z)
|
||||
z = torch.cat(z_list, dim=2)
|
||||
if self.token_drop > 0:
|
||||
z = z[:, :, : -self.token_drop]
|
||||
if self.isolated_first_frame:
|
||||
z_first = self._adaptive_encode(x[:, :, :1, :, :])
|
||||
z = torch.cat([z_first, z], dim=2)
|
||||
if self.isolated_last_frame:
|
||||
frame_num = x.shape[2]
|
||||
last_idx = frame_num - self.frame_drop + offset_frame
|
||||
z_last = self._adaptive_encode(x[:, :, last_idx : last_idx + 1, :, :])
|
||||
z = torch.cat([z, z_last], dim=2)
|
||||
return z
|
||||
|
||||
|
||||
def _video_x(t, c=3, h=4, w=4):
|
||||
return torch.arange(t * c * h * w, dtype=torch.float32).reshape(1, c, t, h, w) / 255.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_frames", [8, 10, 11, 13])
|
||||
@pytest.mark.parametrize(
|
||||
"flags",
|
||||
[
|
||||
{},
|
||||
{"isolated_first": True},
|
||||
{"isolated_last": True},
|
||||
{"isolated_first": True, "isolated_last": True},
|
||||
{"key_frame": True},
|
||||
],
|
||||
)
|
||||
def test_encode_fork_matches_whole_video_pad_bitwise(num_frames, flags):
|
||||
x = _video_x(num_frames)
|
||||
ref_model = _EncodeModel(**flags)
|
||||
expected = ref_model.reference_encode_temporal(x.clone())
|
||||
# Empty-slice encodes (out-of-range isolated-last indexes are no-ops in
|
||||
# the checkpoint) carry no content; compare only real calls.
|
||||
ref_calls = [c.clone() for c in ref_model.encoded if c.shape[2] > 0]
|
||||
|
||||
fork_model = _EncodeModel(**flags)
|
||||
out = _encode_temporal_tail_pad(fork_model, x.clone())
|
||||
|
||||
assert out.shape == expected.shape
|
||||
assert torch.equal(out, expected)
|
||||
assert len(fork_model.encoded) == len(ref_calls)
|
||||
for got, want in zip(fork_model.encoded, ref_calls):
|
||||
assert torch.equal(got, want)
|
||||
|
||||
|
||||
def test_encode_fork_avoids_whole_video_pad_for_misaligned_input():
|
||||
# 375 frames with an asymmetric config (isolated last frame only):
|
||||
# the checkpoint always pads (375 % 17 != 0); the fork must encode the
|
||||
# tail chunk with an in-chunk pad instead of concatenating the video.
|
||||
model = _EncodeModel(isolated_last=True, frame_drop=16)
|
||||
model.clip_length = 17
|
||||
x = _video_x(375, c=1, h=2, w=2)
|
||||
_encode_temporal_tail_pad(model, x)
|
||||
# 375 % 17 == 1 → pad 16 inside the tail chunk; the isolated last frame
|
||||
# resolves to the final real frame (pad frames replicate it).
|
||||
assert len(model.encoded) == (375 // 17) + 1 + 1
|
||||
assert torch.equal(model.encoded[-1].flatten(), x[:, :, -1:, :, :].flatten())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _decode_temporal_streaming uint8 fork
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DENORM_MEAN = (-2.11628412, -2.03571429, -1.80444444)
|
||||
DENORM_STD = (4.36681223, 4.46428571, 4.44444444)
|
||||
|
||||
|
||||
class _FakeDenorm:
|
||||
def __init__(self):
|
||||
self.mean = DENORM_MEAN
|
||||
self.std = DENORM_STD
|
||||
|
||||
|
||||
class _FakeProcessor:
|
||||
def __init__(self):
|
||||
self.transform_rev = _FakeDenorm()
|
||||
|
||||
|
||||
class _DecodeModel:
|
||||
"""Minimal stand-in for the checkpoint's streaming decode contract."""
|
||||
|
||||
def __init__(self, *, token_drop=0, frame_overlap=0):
|
||||
self.tokens_chunk_size = 2
|
||||
self.vae_ratio_t = 1
|
||||
self.token_overlap = 1 if token_drop else 0
|
||||
self.token_drop = token_drop
|
||||
self.frame_pre_padding = 0
|
||||
self.frame_overlap = frame_overlap
|
||||
self.processor = _FakeProcessor()
|
||||
|
||||
def _adaptive_decode(self, clip_z):
|
||||
# Deterministic pseudo-decode: map latent tokens to frames 1:1.
|
||||
return torch.sin(clip_z * 3.0)
|
||||
|
||||
def blend(self, a, b, overlap, dim=-3):
|
||||
return (a + b) / 2.0
|
||||
|
||||
def _decode_temporal_pad_frames(self, z, pad_tokens):
|
||||
return int(pad_tokens)
|
||||
|
||||
def _decode_temporal_output_frame_plan(self, z, z_head, z_tail, num_chunks, pad_tokens):
|
||||
total = num_chunks * self.tokens_chunk_size * self.vae_ratio_t
|
||||
if z_head is not None:
|
||||
total += 1
|
||||
if z_tail is not None:
|
||||
total += 1
|
||||
if self.token_drop > 0:
|
||||
total += self.frame_overlap
|
||||
return int(total), int(pad_tokens), int(total - pad_tokens)
|
||||
|
||||
def legacy_streaming(self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype):
|
||||
"""The checkpoint's original: accumulate in the cat dtype, quantize later."""
|
||||
total_frames, _, output_frames = self._decode_temporal_output_frame_plan(
|
||||
z, z_head, z_tail, num_chunks, pad_tokens
|
||||
)
|
||||
chunk_dec = self.tokens_chunk_size * self.vae_ratio_t
|
||||
split_count = int(self.token_drop > 0) + 1
|
||||
dec = None
|
||||
dec_overlap = None
|
||||
write_pos = 0
|
||||
|
||||
def write_part(part):
|
||||
nonlocal dec, write_pos
|
||||
if dec is None:
|
||||
out_shape = list(part.shape)
|
||||
out_shape[2] = output_frames
|
||||
dec = torch.empty(out_shape, dtype=part.dtype)
|
||||
n = min(part.shape[2], dec.shape[2] - write_pos)
|
||||
if n > 0:
|
||||
dec[:, :, write_pos : write_pos + n].copy_(part[:, :, :n])
|
||||
write_pos += n
|
||||
|
||||
for i in range(num_chunks):
|
||||
t_start = i * self.tokens_chunk_size
|
||||
t_end = t_start + self.tokens_chunk_size + self.token_overlap
|
||||
clip_z = z[:, :, t_start:t_end]
|
||||
clip_dec = self._adaptive_decode(clip_z)
|
||||
if temporal_cat_dtype is not None and clip_dec.dtype != temporal_cat_dtype:
|
||||
clip_dec = clip_dec.to(temporal_cat_dtype)
|
||||
for j in range(split_count):
|
||||
chunk = clip_dec[:, :, j * chunk_dec : min(j * chunk_dec + chunk_dec, clip_dec.shape[2])]
|
||||
if j == 0:
|
||||
if dec_overlap is not None:
|
||||
chunk = self.blend(dec_overlap, chunk, self.frame_overlap, dim=-3)
|
||||
dec_overlap = None
|
||||
write_part(chunk)
|
||||
else:
|
||||
dec_overlap = chunk.contiguous()
|
||||
if i == num_chunks - 1 and dec_overlap is not None:
|
||||
write_part(dec_overlap)
|
||||
dec_overlap = None
|
||||
# Legacy post-decode chain: revert in place in the decoded dtype, then
|
||||
# the adapter's ``frames.float()`` before the pipeline's quantizer.
|
||||
assert dec is not None
|
||||
mean = torch.tensor(DENORM_MEAN, dtype=dec.dtype).view(1, 3, 1, 1, 1)
|
||||
std = torch.tensor(DENORM_STD, dtype=dec.dtype).view(1, 3, 1, 1, 1)
|
||||
dec.sub_(mean).div_(std).clamp_(0.0, 1.0)
|
||||
dec = dec.float()
|
||||
dec.mul_(255.0).round_()
|
||||
return dec.to(torch.uint8)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token_drop,frame_overlap", [(0, 0), (1, 1)])
|
||||
@pytest.mark.parametrize("temporal_cat_dtype", [None, torch.float16, torch.bfloat16])
|
||||
def test_decode_fork_matches_legacy_chain_bitwise(token_drop, frame_overlap, temporal_cat_dtype):
|
||||
# The quantizer must see FP32 no matter the cat dtype: the legacy adapter
|
||||
# returned frames.float() before the pipeline multiplied and rounded, so
|
||||
# an fp16/bf16 cat dtype only narrows denormalization, never the rounding.
|
||||
torch.manual_seed(0)
|
||||
tokens = 2 * 2 + (1 if token_drop else 0) # num_chunks * tokens_chunk_size
|
||||
z = torch.randn(1, 3, tokens, 4, 4)
|
||||
legacy = _DecodeModel(token_drop=token_drop, frame_overlap=frame_overlap)
|
||||
expected = legacy.legacy_streaming(z.clone(), None, None, 2, 0, temporal_cat_dtype)
|
||||
|
||||
fork = _DecodeModel(token_drop=token_drop, frame_overlap=frame_overlap)
|
||||
out = _decode_temporal_streaming_uint8(fork, z.clone(), None, None, 2, 0, temporal_cat_dtype)
|
||||
assert out.dtype == torch.uint8
|
||||
assert out.shape == expected.shape
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
|
||||
def test_decode_fork_rejects_empty_plan():
|
||||
model = _DecodeModel()
|
||||
model.tokens_chunk_size = 0
|
||||
with pytest.raises(ValueError, match="non-positive output_frames"):
|
||||
_decode_temporal_streaming_uint8(model, torch.randn(1, 3, 0, 4, 4), None, None, 0, 0, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# install + escape hatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_install_binds_both_forks():
|
||||
model = _EncodeModel()
|
||||
# cast(Any, ...) keeps mypy from narrowing the probes to the lambda type,
|
||||
# which has no __func__ for the bound-method assertions below.
|
||||
model.encode_temporal = cast(Any, lambda x: x) # placeholder for the capability check
|
||||
model._decode_temporal_streaming = cast(Any, lambda *a: None)
|
||||
model.processor = _FakeProcessor()
|
||||
install_temporal_stream_patches(model)
|
||||
assert model.encode_temporal.__func__ is _encode_temporal_tail_pad
|
||||
assert model._decode_temporal_streaming.__func__ is _decode_temporal_streaming_uint8
|
||||
|
||||
|
||||
def test_install_skips_decode_fork_without_denorm_contract():
|
||||
model = _EncodeModel()
|
||||
model.encode_temporal = cast(Any, lambda x: x)
|
||||
model._decode_temporal_streaming = cast(Any, lambda *a: None)
|
||||
install_temporal_stream_patches(model) # no processor -> keep original
|
||||
assert model.encode_temporal.__func__ is _encode_temporal_tail_pad
|
||||
assert not hasattr(model._decode_temporal_streaming, "__func__")
|
||||
|
||||
|
||||
def test_install_respects_legacy_env(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_OMNI_VAE_LEGACY_TEMPORAL", "1")
|
||||
model = _EncodeModel()
|
||||
install_temporal_stream_patches(model)
|
||||
assert not hasattr(getattr(model, "encode_temporal", None), "__func__")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _prepare_minimax_h3_video_output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prepare_output_matches_previous_conversion():
|
||||
video = torch.randn(1, 3, 4, 6, 8).clamp_(-1, 2)
|
||||
legacy = (
|
||||
video.detach()
|
||||
.float()
|
||||
.clamp(0, 1)
|
||||
.mul(255)
|
||||
.round()
|
||||
.permute(0, 2, 3, 4, 1)
|
||||
.to(dtype=torch.uint8, memory_format=torch.contiguous_format)
|
||||
)
|
||||
out = _prepare_minimax_h3_video_output(video.clone())
|
||||
assert out.dtype == torch.uint8
|
||||
assert out.shape == (1, 4, 6, 8, 3)
|
||||
assert out.is_contiguous()
|
||||
assert torch.equal(out, legacy)
|
||||
|
||||
|
||||
def test_prepare_output_uint8_passthrough():
|
||||
video = torch.randint(0, 256, (1, 3, 4, 6, 8), dtype=torch.uint8)
|
||||
out = _prepare_minimax_h3_video_output(video)
|
||||
assert out.dtype == torch.uint8
|
||||
assert out.is_contiguous()
|
||||
assert torch.equal(out, video.permute(0, 2, 3, 4, 1).contiguous())
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
@@ -0,0 +1,500 @@
|
||||
#!/usr/bin/env python
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""π0.5 parity against LeRobot.
|
||||
|
||||
Reference: **lerobot 0.6.1**, transformers 5.5.4, torch 2.11.0+cpu. The oracle
|
||||
is LeRobot's own code, so a later version changing its π0.5 implementation
|
||||
fails this without anything here regressing — diff ``policies/pi05/`` first.
|
||||
|
||||
Compares final action chunks under fixed noise: float32 exactly, bfloat16
|
||||
against LeRobot's own bfloat16 layout within ``BF16_ATOL``.
|
||||
|
||||
Needs a separate ``lerobot[pi]`` venv (its dependencies conflict with
|
||||
vllm-omni's) with the vllm-omni ``pi05`` package importable::
|
||||
|
||||
python -m pytest tests/diffusion/models/pi05/test_pi05_parity.py -v -s
|
||||
|
||||
Skips itself when LeRobot is absent. ``PI05_PARITY_MODEL_PATH`` points at a
|
||||
local checkpoint to skip the HF download.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# local_model: needs real weights + a lerobot venv, so it runs locally rather
|
||||
# than in the ready-CI. Additionally gated on lerobot being importable.
|
||||
_HAS_LEROBOT = importlib.util.find_spec("lerobot") is not None
|
||||
|
||||
# cpu: the comparison runs on CPU in float32 so both implementations see
|
||||
# identical numerics; local_model: it needs a lerobot venv that CI does not have.
|
||||
pytestmark = [pytest.mark.local_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
# ─── Config (fixed; matches LeRobot defaults for a π0.5 checkpoint) ────
|
||||
DEVICE = "cpu"
|
||||
DTYPE_STR = "float32"
|
||||
ATOL = 1e-4
|
||||
BF16_ATOL = 5e-2 # measured 2.95e-2, reproducible; margin is for version drift
|
||||
NUM_STEPS = 10
|
||||
BATCH_SIZE = 2
|
||||
ACTION_DIM = 32
|
||||
STATE_DIM = 32
|
||||
ACTION_HORIZON = 50
|
||||
MAX_TOKEN_LEN = 200 # π0 uses 48
|
||||
NUM_STATE_BINS = 256
|
||||
|
||||
MODEL_PATH = os.environ.get("PI05_PARITY_MODEL_PATH", "lerobot/pi05_base")
|
||||
|
||||
CAMERAS = ("base_0_rgb", "left_wrist_0_rgb", "right_wrist_0_rgb")
|
||||
|
||||
|
||||
def _resolve_checkpoint_dir() -> str:
|
||||
if os.path.isdir(MODEL_PATH):
|
||||
return MODEL_PATH
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
return snapshot_download(repo_id=MODEL_PATH, repo_type="model")
|
||||
|
||||
|
||||
# ─── Dummy dataset stats ──────────────────────────────────────────────
|
||||
def _dummy_dataset_stats() -> dict:
|
||||
"""Identity-ish stats. π0.5 normalizes with QUANTILES by default, so q01/q99
|
||||
carry the load here where π0's mean/std would.
|
||||
|
||||
Using dummy stats means this test needs no checkpoint with real
|
||||
``norm_stats`` — it verifies numerical equivalence, not calibration.
|
||||
"""
|
||||
return {
|
||||
"observation.state": {
|
||||
"mean": torch.zeros(STATE_DIM),
|
||||
"std": torch.ones(STATE_DIM),
|
||||
"q01": -torch.ones(STATE_DIM),
|
||||
"q99": torch.ones(STATE_DIM),
|
||||
},
|
||||
"action": {
|
||||
"mean": torch.zeros(ACTION_DIM),
|
||||
"std": torch.ones(ACTION_DIM),
|
||||
"q01": -torch.ones(ACTION_DIM),
|
||||
"q99": torch.ones(ACTION_DIM),
|
||||
},
|
||||
"images": {
|
||||
cam: {
|
||||
"mean": torch.zeros(3, 224, 224),
|
||||
"std": torch.ones(3, 224, 224),
|
||||
"q01": torch.zeros(3, 224, 224),
|
||||
"q99": torch.ones(3, 224, 224),
|
||||
}
|
||||
for cam in CAMERAS
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _create_dummy_batch(batch_size: int = BATCH_SIZE, num_views: int = 3, device: str = DEVICE) -> dict:
|
||||
"""Reproducible dummy inputs — identical across both implementations."""
|
||||
g = torch.Generator(device="cpu").manual_seed(0)
|
||||
prompt = "Pick up the red block and place it in the bin"
|
||||
batch = {
|
||||
"observation.state": torch.randn(batch_size, STATE_DIM, generator=g, dtype=torch.float32).to(device),
|
||||
"action": torch.randn(batch_size, ACTION_HORIZON, ACTION_DIM, generator=g, dtype=torch.float32).to(device),
|
||||
"task": [prompt for _ in range(batch_size)],
|
||||
}
|
||||
for cam in CAMERAS[:num_views]:
|
||||
batch[f"observation.images.{cam}"] = torch.rand(batch_size, 3, 224, 224, generator=g, dtype=torch.float32).to(
|
||||
device
|
||||
)
|
||||
return batch
|
||||
|
||||
|
||||
# ─── Instantiation ────────────────────────────────────────────────────
|
||||
def _instantiate_lerobot(use_relative_actions: bool = False, action_feature_names=None, dtype: str = DTYPE_STR):
|
||||
from lerobot.policies.pi05 import PI05Policy
|
||||
from lerobot.policies.pi05.processor_pi05 import make_pi05_pre_post_processors
|
||||
|
||||
policy = PI05Policy.from_pretrained(MODEL_PATH, strict=True, dtype=dtype)
|
||||
policy.to(DEVICE)
|
||||
policy.config.device = DEVICE
|
||||
policy.config.use_relative_actions = use_relative_actions
|
||||
if action_feature_names is not None:
|
||||
# Must land on the config *before* the processors are built:
|
||||
# make_pi05_pre_post_processors captures action_feature_names into
|
||||
# RelativeActionsProcessorStep at construction time, and its _build_mask
|
||||
# treats ``action_names is None`` as "shift every dim" — including the
|
||||
# gripper that relative_exclude_joints is supposed to hold back. Setting
|
||||
# it afterwards leaves the step with the all-True mask and silently
|
||||
# compares two different transforms.
|
||||
policy.config.action_feature_names = action_feature_names
|
||||
policy.eval()
|
||||
|
||||
pre, post = make_pi05_pre_post_processors(config=policy.config, dataset_stats=_dummy_dataset_stats())
|
||||
return policy, pre, post
|
||||
|
||||
|
||||
def _instantiate_vllm_omni(
|
||||
use_relative_actions: bool = False,
|
||||
action_feature_names=None,
|
||||
dtype: str = DTYPE_STR,
|
||||
):
|
||||
"""Build the vllm-omni π0.5 model in isolation (no pipeline, no engine)."""
|
||||
from vllm_omni.diffusion.models.pi05 import Pi05Config, Pi05ForActionPrediction
|
||||
from vllm_omni.diffusion.models.pi05.pipeline_pi05 import _set_inference_dtype
|
||||
|
||||
cfg = Pi05Config(
|
||||
max_action_dim=ACTION_DIM,
|
||||
max_state_dim=STATE_DIM,
|
||||
chunk_size=ACTION_HORIZON,
|
||||
num_inference_steps=NUM_STEPS,
|
||||
tokenizer_max_length=MAX_TOKEN_LEN,
|
||||
state_num_bins=NUM_STATE_BINS,
|
||||
dtype=dtype,
|
||||
use_relative_actions=use_relative_actions,
|
||||
relative_exclude_joints=["gripper"] if action_feature_names else [],
|
||||
action_feature_names=action_feature_names,
|
||||
)
|
||||
model = Pi05ForActionPrediction(cfg)
|
||||
_set_inference_dtype(model, getattr(torch, dtype))
|
||||
model.to(device=DEVICE)
|
||||
_load_lerobot_weights(model)
|
||||
model.eval()
|
||||
return model, cfg
|
||||
|
||||
|
||||
def _load_lerobot_weights(model):
|
||||
import safetensors.torch
|
||||
|
||||
path = os.path.join(_resolve_checkpoint_dir(), "model.safetensors")
|
||||
state = safetensors.torch.load_file(path)
|
||||
filled = model.load_weights(list(state.items()))
|
||||
assert filled, "no weights were loaded — the remap rules are broken"
|
||||
|
||||
|
||||
def _extract_lerobot_model_inputs(lerobot_policy, processed_batch):
|
||||
"""Mimic what ``PI05Policy.predict_action_chunk`` feeds into
|
||||
``self.model.sample_actions``. Using these *exact* tensors for vllm-omni
|
||||
means any divergence must come from the core model, not preprocessing.
|
||||
|
||||
Note π0.5 has no ``prepare_state`` step: the state is already inside
|
||||
``lang_tokens``.
|
||||
"""
|
||||
from lerobot.utils.constants import (
|
||||
OBS_LANGUAGE_ATTENTION_MASK,
|
||||
OBS_LANGUAGE_TOKENS,
|
||||
)
|
||||
|
||||
images, img_masks = lerobot_policy._preprocess_images(processed_batch)
|
||||
return images, img_masks, processed_batch[OBS_LANGUAGE_TOKENS], processed_batch[OBS_LANGUAGE_ATTENTION_MASK]
|
||||
|
||||
|
||||
def _make_fixed_noise(batch_size: int, device: str) -> torch.Tensor:
|
||||
g = torch.Generator(device="cpu").manual_seed(42)
|
||||
return torch.randn(batch_size, ACTION_HORIZON, ACTION_DIM, generator=g, dtype=torch.float32).to(device)
|
||||
|
||||
|
||||
# ─── Main parity test ─────────────────────────────────────────────────
|
||||
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
|
||||
@pytest.mark.parametrize("num_views", [1, 2, 3])
|
||||
def test_pi05_vllm_omni_vs_lerobot(num_views):
|
||||
lerobot_policy, lerobot_pre, _ = _instantiate_lerobot()
|
||||
omni_model, _ = _instantiate_vllm_omni()
|
||||
|
||||
raw_batch = _create_dummy_batch(num_views=num_views)
|
||||
processed = lerobot_pre(copy.deepcopy(raw_batch))
|
||||
images, img_masks, lang_tokens, lang_masks = _extract_lerobot_model_inputs(lerobot_policy, processed)
|
||||
noise = _make_fixed_noise(raw_batch["observation.state"].shape[0], DEVICE)
|
||||
|
||||
# The input contract: 256 tokens per view + a constant 200 text tokens.
|
||||
assert lang_tokens.shape[1] == MAX_TOKEN_LEN, "π0.5 must pad text to exactly 200 tokens"
|
||||
|
||||
with torch.no_grad():
|
||||
lerobot_actions = lerobot_policy.model.sample_actions(
|
||||
images, img_masks, lang_tokens, lang_masks, noise=noise, num_steps=NUM_STEPS
|
||||
)
|
||||
omni_actions = omni_model.sample_actions(
|
||||
images=images,
|
||||
image_masks=img_masks,
|
||||
lang_tokens=lang_tokens,
|
||||
lang_masks=lang_masks,
|
||||
noise=noise,
|
||||
num_steps=NUM_STEPS,
|
||||
)
|
||||
|
||||
diff = (lerobot_actions.float() - omni_actions.float()).abs()
|
||||
print(f"[parity] views={num_views} |Δ| max={diff.max().item():.2e} mean={diff.mean().item():.2e}")
|
||||
if not torch.allclose(lerobot_actions.float(), omni_actions.float(), atol=ATOL):
|
||||
_diagnose_divergence(lerobot_policy.model, omni_model, images, img_masks, lang_tokens, lang_masks, noise)
|
||||
assert torch.allclose(lerobot_actions.float(), omni_actions.float(), atol=ATOL), (
|
||||
f"actions differ beyond atol={ATOL}; max_diff={diff.max().item():.2e}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
|
||||
def test_pi05_bfloat16_matches_lerobot_bfloat16():
|
||||
"""Both sides in bfloat16, so this compares implementations rather than
|
||||
measuring bfloat16's drift from float32."""
|
||||
# LeRobot applies the layout in PaliGemmaWithExpertModel.__init__ from
|
||||
# config.dtype, so the policy must be built as bfloat16 rather than cast after.
|
||||
lerobot_policy, lerobot_pre, _ = _instantiate_lerobot(dtype="bfloat16")
|
||||
|
||||
omni_model, _ = _instantiate_vllm_omni(dtype="bfloat16")
|
||||
|
||||
raw_batch = _create_dummy_batch(num_views=3)
|
||||
processed = lerobot_pre(copy.deepcopy(raw_batch))
|
||||
images, img_masks, lang_tokens, lang_masks = _extract_lerobot_model_inputs(lerobot_policy, processed)
|
||||
noise = _make_fixed_noise(raw_batch["observation.state"].shape[0], DEVICE)
|
||||
|
||||
with torch.no_grad():
|
||||
lerobot_actions = lerobot_policy.model.sample_actions(
|
||||
images, img_masks, lang_tokens, lang_masks, noise=noise, num_steps=NUM_STEPS
|
||||
)
|
||||
omni_actions = omni_model.sample_actions(
|
||||
images=images,
|
||||
image_masks=img_masks,
|
||||
lang_tokens=lang_tokens,
|
||||
lang_masks=lang_masks,
|
||||
noise=noise,
|
||||
num_steps=NUM_STEPS,
|
||||
)
|
||||
|
||||
diff = (lerobot_actions.float() - omni_actions.float()).abs()
|
||||
print(f"[parity] bfloat16 |Δ| max={diff.max().item():.2e} mean={diff.mean().item():.2e}")
|
||||
assert torch.allclose(lerobot_actions.float(), omni_actions.float(), atol=BF16_ATOL), (
|
||||
f"bfloat16 actions differ beyond atol={BF16_ATOL}; max_diff={diff.max().item():.2e}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
|
||||
def test_pi05_prompt_parity():
|
||||
"""π0.5's state is carried as prompt tokens, so the tokenized prompt is
|
||||
itself part of the contract. A bin-index mismatch here (e.g. from running
|
||||
the discretizer before the normalizer) would otherwise only surface as a
|
||||
small, plausible-looking action difference.
|
||||
"""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm_omni.diffusion.models.pi05.processor_pi05 import (
|
||||
apply_norm,
|
||||
build_norm_stats,
|
||||
build_pi05_prompt,
|
||||
tokenize_prompt,
|
||||
)
|
||||
|
||||
lerobot_policy, lerobot_pre, _ = _instantiate_lerobot()
|
||||
raw_batch = _create_dummy_batch(batch_size=1)
|
||||
processed = lerobot_pre(copy.deepcopy(raw_batch))
|
||||
|
||||
from lerobot.utils.constants import OBS_LANGUAGE_TOKENS
|
||||
|
||||
lerobot_tokens = processed[OBS_LANGUAGE_TOKENS][0]
|
||||
|
||||
stats = _dummy_dataset_stats()["observation.state"]
|
||||
norm = build_norm_stats({"state": {"q01": stats["q01"], "q99": stats["q99"]}}, "state")
|
||||
normalized = apply_norm(raw_batch["observation.state"][0].float(), norm)
|
||||
prompt = build_pi05_prompt(
|
||||
task=raw_batch["task"][0],
|
||||
normalized_state=normalized.numpy(),
|
||||
state_num_bins=NUM_STATE_BINS,
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained("google/paligemma-3b-pt-224", padding_side="right")
|
||||
ids, _ = tokenize_prompt(tokenizer, prompt, MAX_TOKEN_LEN)
|
||||
|
||||
assert len(ids) == MAX_TOKEN_LEN
|
||||
assert torch.equal(torch.tensor(ids), lerobot_tokens.cpu()), "π0.5 prompt tokens diverge from LeRobot"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
|
||||
@pytest.mark.parametrize("state_dim", [7, STATE_DIM])
|
||||
def test_pi05_prompt_parity_across_state_widths(state_dim):
|
||||
"""``pi05_base`` declares a 32-dim state, which is exactly ``max_state_dim``,
|
||||
so the full-policy parity above cannot tell padding apart from the real
|
||||
width. This drives LeRobot's tokenizer step directly, at both widths.
|
||||
"""
|
||||
from lerobot.lerobot_types import TransitionKey
|
||||
from lerobot.policies.pi05.processor_pi05 import Pi05PrepareStateTokenizerProcessorStep
|
||||
from lerobot.utils.constants import OBS_STATE
|
||||
|
||||
from vllm_omni.diffusion.models.pi05.processor_pi05 import build_pi05_prompt
|
||||
|
||||
g = torch.Generator().manual_seed(state_dim)
|
||||
state = torch.randn(state_dim, generator=g, dtype=torch.float32)
|
||||
task = "pick up the red block"
|
||||
|
||||
step = Pi05PrepareStateTokenizerProcessorStep(max_state_dim=STATE_DIM)
|
||||
transition = step(
|
||||
{
|
||||
TransitionKey.OBSERVATION: {OBS_STATE: state.unsqueeze(0)},
|
||||
TransitionKey.COMPLEMENTARY_DATA: {"task": [task]},
|
||||
}
|
||||
)
|
||||
lerobot_prompt = transition[TransitionKey.COMPLEMENTARY_DATA]["task"][0]
|
||||
|
||||
prompt = build_pi05_prompt(task=task, normalized_state=state.numpy())
|
||||
|
||||
assert prompt == lerobot_prompt
|
||||
assert len(prompt.split("State: ")[1].split(";")[0].split()) == state_dim
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
|
||||
@pytest.mark.parametrize(
|
||||
"lerobot_mode,stats,ours,state,state_dim",
|
||||
[
|
||||
("MEAN_STD", {"mean": [0.0], "std": [1e-7]}, {"mode": "mean_std", "mean": [0.0], "std": [1e-7]}, [1e-7], 1),
|
||||
("QUANTILES", {"q01": [0.0], "q99": [1e-7]}, {"mode": "quantile", "q01": [0.0], "q99": [1e-7]}, [1e-7], 1),
|
||||
("QUANTILES", {"q01": [2.0], "q99": [2.0]}, {"mode": "quantile", "q01": [2.0], "q99": [2.0]}, [2.0], 1),
|
||||
(
|
||||
"QUANTILES",
|
||||
{"q01": [0.0] * 4, "q99": [10.0] * 4},
|
||||
{"mode": "quantile", "q01": [0.0] * 4, "q99": [10.0] * 4},
|
||||
[-10.0, 0.0, 5.0, 20.0],
|
||||
4,
|
||||
),
|
||||
(
|
||||
"MEAN_STD",
|
||||
{"mean": [1.0, 2.0], "std": [0.5, 4.0]},
|
||||
{"mode": "mean_std", "mean": [1.0, 2.0], "std": [0.5, 4.0]},
|
||||
[3.0, -6.0],
|
||||
2,
|
||||
),
|
||||
("MIN_MAX", {"min": [-1.0], "max": [3.0]}, {"mode": "min_max", "min": [-1.0], "max": [3.0]}, [2.0], 1),
|
||||
],
|
||||
)
|
||||
def test_pi05_state_normalization_parity(lerobot_mode, stats, ours, state, state_dim):
|
||||
"""State normalization against LeRobot's ``NormalizerProcessorStep``.
|
||||
|
||||
The full-policy parity test feeds both sides stats with a wide range, so it
|
||||
cannot see the denominator rule. A wrongly normalized state still discretizes
|
||||
into a well-formed prompt: for ``q01=0, q99=1e-7`` a threshold substitution
|
||||
moves the state token from bin 255 to bin 0.
|
||||
"""
|
||||
from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
|
||||
from lerobot.processor.normalize_processor import NormalizerProcessorStep
|
||||
from lerobot.utils.constants import OBS_STATE
|
||||
|
||||
from vllm_omni.diffusion.models.pi05.processor_pi05 import apply_norm, build_norm_stats
|
||||
|
||||
step = NormalizerProcessorStep(
|
||||
features={OBS_STATE: PolicyFeature(type=FeatureType.STATE, shape=(state_dim,))},
|
||||
norm_map={FeatureType.STATE: getattr(NormalizationMode, lerobot_mode)},
|
||||
stats={OBS_STATE: {k: torch.tensor(v, dtype=torch.float32) for k, v in stats.items()}},
|
||||
)
|
||||
expected = step._apply_transform(
|
||||
torch.tensor(state, dtype=torch.float32), OBS_STATE, FeatureType.STATE, inverse=False
|
||||
)
|
||||
|
||||
actual = apply_norm(torch.tensor(state, dtype=torch.float32), build_norm_stats({"state": ours}, "state"))
|
||||
|
||||
assert torch.allclose(actual, expected, atol=1e-6)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_LEROBOT, reason="lerobot not installed (run in a lerobot venv).")
|
||||
def test_pi05_relative_actions_parity():
|
||||
"""With ``use_relative_actions=True`` the checkpoint's norm_stats live in
|
||||
relative space, so the post-pipeline must add the raw state back. Compares
|
||||
the full pre → model → post path against LeRobot's."""
|
||||
from lerobot.utils.constants import (
|
||||
OBS_LANGUAGE_ATTENTION_MASK,
|
||||
OBS_LANGUAGE_TOKENS,
|
||||
)
|
||||
|
||||
from vllm_omni.diffusion.models.pi05.processor_pi05 import (
|
||||
Pi05RelativeActions,
|
||||
apply_norm,
|
||||
build_norm_stats,
|
||||
)
|
||||
|
||||
action_names = [f"joint_{i}" for i in range(ACTION_DIM - 1)] + ["gripper"]
|
||||
lerobot_policy, lerobot_pre, lerobot_post = _instantiate_lerobot(
|
||||
use_relative_actions=True, action_feature_names=action_names
|
||||
)
|
||||
omni_model, cfg = _instantiate_vllm_omni(use_relative_actions=True, action_feature_names=action_names)
|
||||
|
||||
raw_batch = _create_dummy_batch()
|
||||
processed = lerobot_pre(copy.deepcopy(raw_batch))
|
||||
images, img_masks = lerobot_policy._preprocess_images(processed)
|
||||
lang_tokens = processed[OBS_LANGUAGE_TOKENS]
|
||||
lang_masks = processed[OBS_LANGUAGE_ATTENTION_MASK]
|
||||
noise = _make_fixed_noise(raw_batch["observation.state"].shape[0], DEVICE)
|
||||
|
||||
with torch.no_grad():
|
||||
lerobot_actions = lerobot_post(
|
||||
lerobot_policy.model.sample_actions(
|
||||
images, img_masks, lang_tokens, lang_masks, noise=noise, num_steps=NUM_STEPS
|
||||
)
|
||||
)
|
||||
omni_raw = omni_model.sample_actions(
|
||||
images=images,
|
||||
image_masks=img_masks,
|
||||
lang_tokens=lang_tokens,
|
||||
lang_masks=lang_masks,
|
||||
noise=noise,
|
||||
num_steps=NUM_STEPS,
|
||||
)
|
||||
|
||||
rel = Pi05RelativeActions(
|
||||
enabled=True,
|
||||
exclude_joints=cfg.relative_exclude_joints,
|
||||
action_names=action_names,
|
||||
max_action_dim=ACTION_DIM,
|
||||
)
|
||||
omni_actions = apply_norm(omni_raw, build_norm_stats(cfg.norm_stats, "action"), inverse=True)
|
||||
# Per-sample reference state, matching LeRobot: its
|
||||
# RelativeActionsProcessorStep caches the whole (B, D) state on the input
|
||||
# side and shifts each sample by its own row. Passing a single row here
|
||||
# would shift every sample by sample 0's state, which only agrees for B=1.
|
||||
omni_actions = rel.to_absolute(omni_actions, raw_batch["observation.state"])
|
||||
|
||||
lerobot_actions = torch.as_tensor(lerobot_actions).float()
|
||||
assert torch.allclose(lerobot_actions, omni_actions.float(), atol=ATOL), (
|
||||
"relative-action path diverges from LeRobot; "
|
||||
f"max_diff={(lerobot_actions - omni_actions.float()).abs().max().item():.2e}"
|
||||
)
|
||||
|
||||
|
||||
# ─── Per-stage divergence diagnostics ─────────────────────────────────
|
||||
@torch.no_grad()
|
||||
def _diagnose_divergence(lerobot_flow_model, omni_model, images, img_masks, lang_tokens, lang_masks, noise):
|
||||
"""Localize a numerical mismatch to a specific stage:
|
||||
1. Prefix embeddings (SigLIP / embed_tokens / projector) — image vs lang.
|
||||
2. Prefix KV cache layer 0 — PaliGemma LM attention.
|
||||
3. The AdaRMS timestep conditioning vector.
|
||||
4. A single denoise_step velocity at t=1.0 — action expert forward.
|
||||
"""
|
||||
from vllm_omni.diffusion.models.pi05.modeling_pi05 import (
|
||||
make_att_2d_masks,
|
||||
prepare_attention_masks_4d,
|
||||
)
|
||||
|
||||
lr_embs, lr_pad, lr_att = lerobot_flow_model.embed_prefix(images, img_masks, lang_tokens, lang_masks)
|
||||
sg_embs, sg_pad, sg_att = omni_model.embed_prefix(images, img_masks, lang_tokens, lang_masks)
|
||||
print(f"[diag] prefix_embs max|Δ| = {(lr_embs.float() - sg_embs.float()).abs().max().item():.2e}")
|
||||
print(f"[diag] prefix_pad_masks equal: {torch.equal(lr_pad, sg_pad)}")
|
||||
|
||||
img_len = 256 * len(images)
|
||||
print(f"[diag] image slice max|Δ| = {(lr_embs[:, :img_len] - sg_embs[:, :img_len]).abs().max().item():.2e}")
|
||||
print(f"[diag] lang slice max|Δ| = {(lr_embs[:, img_len:] - sg_embs[:, img_len:]).abs().max().item():.2e}")
|
||||
|
||||
prefix_att_4d = prepare_attention_masks_4d(make_att_2d_masks(sg_pad, sg_att))
|
||||
prefix_pos = torch.cumsum(sg_pad, dim=1) - 1
|
||||
_, sg_kv = omni_model.paligemma_with_expert.forward(
|
||||
attention_mask=prefix_att_4d,
|
||||
position_ids=prefix_pos,
|
||||
past_key_values=None,
|
||||
inputs_embeds=[sg_embs, None],
|
||||
use_cache=True,
|
||||
)
|
||||
|
||||
t = torch.ones(noise.shape[0], dtype=torch.float32)
|
||||
sg_cond = omni_model.embed_timestep(t)
|
||||
print(f"[diag] adarms cond: shape={tuple(sg_cond.shape)} mean={sg_cond.mean().item():+.4f}")
|
||||
|
||||
sg_vt = omni_model.denoise_step(sg_pad, sg_kv, noise, t)
|
||||
print(f"[diag] denoise_step(t=1) v_t mean={sg_vt.mean().item():+.6f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_pi05_vllm_omni_vs_lerobot(3)
|
||||
Loaded 100 of 769 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user