anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI]Remove DreamID-Omni and MagiHuman support (#6357)

Browse Source 
Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wangyuandCursor authored and GitHub committed 2026-08-26 01:31:12 +00:00
1 parent 40dcfdbb43
commit aeb348d8f8
31 files changed
+26 -6957
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/diffusion/cache/test_cache_backends.py
Vendored
+3 -6
View File
@@ -365,13 +365,10 @@ class TestCacheDiTBackend:
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_enable_dreamid_pipeline_uses_fused_blocks(self, mock_cache_dit, mock_block_adapter):
"""Test DreamID uses pipeline.transformer for cache enable/refresh.
NOTE: DreamID no longer has a custom enabler, so this tests against the generic path.
"""
def test_enable_pipeline_uses_fused_blocks(self, mock_cache_dit, mock_block_adapter):
"""Generic cache path should pick up ``transformer.fused_blocks``."""
mock_pipeline = Mock()
mock_pipeline.__class__.__name__ = "DreamIDOmniPipeline"
mock_pipeline.__class__.__name__ = "FakeFusedBlocksPipeline"
mock_pipeline.transformer = Mock()
mock_pipeline.transformer.fused_blocks = Mock()
mock_pipeline.transformer._cache_dit_adapter_config = CacheDiTAdapterConfig(
tests/diffusion/cache/test_cache_dit.py
Vendored
-10
View File
@@ -6,7 +6,6 @@ Model specific tests for CacheDiT enablement.
"""
import ast
import sys
from pathlib import Path
from unittest.mock import Mock, patch
@@ -25,18 +24,9 @@ from vllm_omni.diffusion.models.longcat_image.longcat_image_transformer import L
from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel
from vllm_omni.platforms import current_omni_platform
# NOTE: We patch DreamID Omni's modules here with mocks so that we can import and inspect
# the class even though the dependency may not be set up correctly; this is ok for these
# tests because we just inspect it and never initialize the model.
for mod in ("dreamid_omni", "dreamid_omni.modules", "dreamid_omni.modules.model"):
sys.modules.setdefault(mod, Mock())
# isort: split
from vllm_omni.diffusion.models.dreamid_omni.fusion import FusionModel as DreamIdOmniModel # noqa: E402
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
SEPARATE_CFG_TRANSFORMERS = [
DreamIdOmniModel,
HeliosTransformer3DModel,
LongCatImageTransformer2DModel,
Cosmos3VFMTransformer,
tests/diffusion/models/dreamid_omni/test_dreamid_omni_hsdp.py
-27
View File
@@ -1,27 +0,0 @@
import pytest
import torch.nn as nn
pytest.importorskip("dreamid_omni.modules.model")
from vllm_omni.diffusion.models.dreamid_omni.fusion import FusionModel
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_dreamid_omni_exposes_hsdp_shard_conditions_for_fused_blocks():
model = object.__new__(FusionModel)
nn.Module.__init__(model)
model.fused_blocks = nn.ModuleList([nn.Linear(4, 4) for _ in range(2)])
model.attn = nn.Identity()
conditions = getattr(model, "_hsdp_shard_conditions", None)
assert conditions is not None
assert len(conditions) == 1
matched = []
for name, module in model.named_modules():
if any(cond(name, module) for cond in conditions):
matched.append(name)
assert matched == ["fused_blocks.0", "fused_blocks.1"]
tests/diffusion/test_diffusion_ipc.py
+1 -1
View File
@@ -181,7 +181,7 @@ def test_diffusion_output_dict_tensors_round_trip_through_shm() -> None:
def test_diffusion_output_tuple_tensors_round_trip_through_shm() -> None:
# LTX2 / DreamID return (video, audio) tuples as DiffusionOutput.output.
# LTX2 returns (video, audio) tuples as DiffusionOutput.output.
video = torch.arange(300_000, dtype=torch.float32)
audio = torch.arange(300_000, dtype=torch.float32) * 3
output = DiffusionOutput(output=(video, audio))
tests/e2e/offline_inference/test_dreamid_omni_cache_dit.py
-290
View File
@@ -1,290 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E test for DreamID-Omni offline inference with Cache-Dit.
Uses the official DreamID-Omni ``test_case/twoip`` example assets instead of
synthetic inputs so the coverage matches the upstream demo structure.
"""
from __future__ import annotations
import importlib.util
import json
import os
import re
import time
from functools import lru_cache
from pathlib import Path
from typing import NamedTuple
import numpy as np
import pytest
import soundfile as sf
from PIL import Image
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
pytestmark = [pytest.mark.diffusion, pytest.mark.cache, pytest.mark.slow]
REPO_ROOT = Path(__file__).resolve().parents[3]
DEFAULT_MODEL_PATH = REPO_ROOT / "dreamid_omni"
MODEL_PATH_ENV = "DREAMID_OMNI_MODEL_PATH"
EXAMPLE_ROOT_ENV = "DREAMID_OMNI_EXAMPLE_ROOT"
NUM_INFERENCE_STEPS = 45
CACHE_CONFIG = {
"Fn_compute_blocks": 1,
"Bn_compute_blocks": 0,
"max_warmup_steps": 4,
"max_cached_steps": 20,
"residual_diff_threshold": 0.24,
"max_continuous_cached_steps": 3,
"enable_taylorseer": False,
"taylorseer_order": 1,
"scm_steps_mask_policy": None,
"scm_steps_policy": "dynamic",
}
class DreamIDGenerationResult(NamedTuple):
video: np.ndarray
audio: np.ndarray
fps: int
duration_s: float
elapsed_s: float
class DreamIDExampleCase(NamedTuple):
prompt: str
image_paths: tuple[Path, ...]
audio_paths: tuple[Path, ...]
def _require_model_path() -> str:
model_path = os.environ.get(MODEL_PATH_ENV, str(DEFAULT_MODEL_PATH))
if not Path(model_path).exists():
pytest.skip(f"DreamID-Omni model bundle not found at {model_path}. Set {MODEL_PATH_ENV}.")
if importlib.util.find_spec("dreamid_omni") is None:
pytest.skip(
"dreamid_omni dependency package is not importable. "
"Run examples/offline_inference/x_to_video_audio/download_dreamid_omni.py first."
)
return model_path
def _dreamid_repo_root_candidates() -> list[Path]:
candidates: list[Path] = []
example_root = os.environ.get(EXAMPLE_ROOT_ENV)
if example_root:
candidates.append(Path(example_root).expanduser().resolve())
spec = importlib.util.find_spec("dreamid_omni")
if spec is not None and spec.origin:
candidates.append(Path(spec.origin).resolve().parents[1] / "test_case" / "twoip")
return candidates
@lru_cache(maxsize=1)
def _load_example_case() -> DreamIDExampleCase:
for candidate_root in _dreamid_repo_root_candidates():
caption_path = candidate_root / "captions" / "20.json"
image_paths = (
candidate_root / "imgs" / "20" / "0.png",
candidate_root / "imgs" / "20" / "1.png",
)
audio_paths = (
candidate_root / "audios" / "20" / "0.wav",
candidate_root / "audios" / "20" / "1.wav",
)
if caption_path.exists() and all(path.exists() for path in image_paths + audio_paths):
prompt = _load_prompt_from_json(caption_path)
return DreamIDExampleCase(
prompt=prompt,
image_paths=image_paths,
audio_paths=audio_paths,
)
searched = ", ".join(str(path) for path in _dreamid_repo_root_candidates()) or "<none>"
pytest.skip(
"DreamID-Omni official example assets not found. "
f"Set {EXAMPLE_ROOT_ENV} to the local 'test_case/twoip' directory. "
f"Searched: {searched}"
)
def _load_prompt_from_json(path: Path) -> str:
raw_text = path.read_text(encoding="utf-8").strip()
parsed = json.loads(raw_text)
if isinstance(parsed, str):
prompt = parsed
elif isinstance(parsed, dict) and isinstance(parsed.get("content"), str):
prompt = parsed["content"]
else:
raise AssertionError(f"Unexpected DreamID caption format in {path}")
prompt = re.sub(
r"\[SPEAKER_TIMESTAMPS_START\].*?\[SPEAKER_TIMESTAMPS_END\]",
"",
prompt,
flags=re.DOTALL,
).strip()
prompt = re.sub(
r"\[AUDIO_DESCRIPTION_START].*?\[AUDIO_DESCRIPTION_END]",
"",
prompt,
flags=re.DOTALL,
).strip()
prompt = re.sub(r"\[[A-Z_]+\]", "", prompt)
prompt = re.sub(r"\n\s*\n", "\n", prompt).strip()
return prompt
def _load_example_images(example_case: DreamIDExampleCase) -> list[Image.Image]:
images: list[Image.Image] = []
for path in example_case.image_paths:
with Image.open(path) as image:
images.append(image.convert("RGB"))
return images
def _load_example_audios(example_case: DreamIDExampleCase) -> list[np.ndarray]:
audios: list[np.ndarray] = []
for path in example_case.audio_paths:
audio_array, sample_rate = sf.read(str(path), dtype="float32")
audio_np = np.asarray(audio_array, dtype=np.float32)
if audio_np.ndim == 2:
audio_np = np.mean(audio_np, axis=1)
audio_np = audio_np[int(sample_rate * 1) : int(sample_rate * 3)]
audios.append(audio_np.reshape(-1))
return audios
def _sampling_params(num_inference_steps: int) -> OmniDiffusionSamplingParams:
return OmniDiffusionSamplingParams(
height=704,
width=1280,
num_inference_steps=num_inference_steps,
seed=42,
extra_args={
"solver_name": "unipc",
"shift": 5.0,
},
)
def _request_payload(example_case: DreamIDExampleCase) -> dict[str, object]:
return {
"prompt": example_case.prompt,
"video_negative_prompt": "jitter, bad hands, blur, distortion",
"audio_negative_prompt": "robotic, muffled, echo, distorted",
"multi_modal_data": {
"image": _load_example_images(example_case),
"audio": _load_example_audios(example_case),
},
}
def _extract_generation(outputs: list[object], elapsed_s: float) -> DreamIDGenerationResult:
"""Extract video and audio from raw Omni outputs"""
assert outputs, "DreamID-Omni returned no outputs"
result = outputs[0]
assert hasattr(result, "images") and result.images, "not return video frames"
video = result.images[0]
assert isinstance(video, np.ndarray), f"Expected video ndarray, got {type(video)}"
assert video.ndim == 4, f"Expected 4D DreamID video tensor, got {video.shape}"
multimodal_output = getattr(result, "multimodal_output", None) or {}
assert "audio" in multimodal_output, "not return audio"
audio = np.asarray(multimodal_output["audio"], dtype=np.float32).reshape(-1)
assert audio.size > 0, "returned empty audio"
audio_sample_rate = int(multimodal_output.get("audio_sample_rate", 16000))
fps = multimodal_output.get("fps", 24)
duration_s = float(audio.size) / float(audio_sample_rate)
generation = DreamIDGenerationResult(
video=video.astype(np.float32, copy=False),
audio=audio,
fps=fps,
duration_s=duration_s,
elapsed_s=elapsed_s,
)
return generation
def _run_generation(
*,
model_path: str,
example_case: DreamIDExampleCase,
use_cache_dit: bool,
) -> DreamIDGenerationResult:
runner_kwargs = {
"model_type": "dreamid-omni",
"parallel_config": DiffusionParallelConfig(cfg_parallel_size=1),
}
if use_cache_dit:
runner_kwargs["cache_backend"] = "cache_dit"
runner_kwargs["cache_config"] = CACHE_CONFIG
with OmniRunner(model_path, **runner_kwargs) as runner:
warmup_payload = _request_payload(example_case)
_ = runner.omni.generate(warmup_payload, _sampling_params(2))
elapsed_samples: list[float] = []
extracted_result: DreamIDGenerationResult | None = None
request_payload = _request_payload(example_case)
start = time.perf_counter()
outputs = runner.omni.generate(request_payload, _sampling_params(NUM_INFERENCE_STEPS))
elapsed_s = time.perf_counter() - start
elapsed_samples.append(elapsed_s)
if extracted_result is None:
extracted_result = _extract_generation(outputs, elapsed_s)
assert extracted_result is not None
median_elapsed_s = float(np.median(np.asarray(elapsed_samples, dtype=np.float64)))
return extracted_result._replace(elapsed_s=median_elapsed_s)
def _video_mae(a: np.ndarray, b: np.ndarray) -> float:
assert a.shape == b.shape, f"Video shapes differ: {a.shape} vs {b.shape}"
return float(np.mean(np.abs(a - b)))
def _audio_rms(audio: np.ndarray) -> float:
audio = np.asarray(audio, dtype=np.float32).reshape(-1)
return float(np.sqrt(np.mean(audio**2)))
def _audio_rms_ratio(a: DreamIDGenerationResult, b: DreamIDGenerationResult) -> float:
rms_a = _audio_rms(a.audio)
rms_b = _audio_rms(b.audio)
return rms_b / max(rms_a, 1e-6)
@hardware_test(res={"cuda": "H100"})
def test_dreamid_omni_cache_dit_e2e() -> None:
model_path = _require_model_path()
example_case = _load_example_case()
baseline = _run_generation(model_path=model_path, example_case=example_case, use_cache_dit=False)
cached = _run_generation(model_path=model_path, example_case=example_case, use_cache_dit=True)
video_mae = _video_mae(baseline.video, cached.video)
audio_rms_ratio = _audio_rms_ratio(baseline, cached)
speedup = baseline.elapsed_s / cached.elapsed_s if cached.elapsed_s > 0 else float("inf")
assert video_mae <= 0.15, f"video drift too much: {video_mae:.6f}"
assert 0.4 <= audio_rms_ratio <= 2.5, f"audio quality regressed too much: rms_ratio={audio_rms_ratio:.6f}"
assert speedup >= 1.05, f"cache-dit improve not enough: {speedup:.3f}x"
tests/e2e/offline_inference/test_magi_human_expansion.py
-162
View File
@@ -1,162 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for MagiHuman pipeline via vLLM-Omni."""
import io
import av
import numpy as np
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "SII-GAIR/daVinci-MagiHuman-Base-1080p"
# (model, stage_config_path, extra_omni_kwargs) — see ``omni_runner`` in tests.helpers.fixtures.runtime
_OMNI_RUNNER_PARAM = (
MODEL,
None,
{
"init_timeout": 1200,
"tensor_parallel_size": 2,
"vae_use_tiling": True,
},
)
_OMNI_RUNNER_PARAM_OFFLOAD = (
MODEL,
None,
{
"init_timeout": 1200,
"tensor_parallel_size": 2,
"enable_cpu_offload": True,
"vae_use_tiling": True,
},
)
pytestmark = [
pytest.mark.slow,
pytest.mark.diffusion,
pytest.mark.parametrize(
"omni_runner",
[_OMNI_RUNNER_PARAM, _OMNI_RUNNER_PARAM_OFFLOAD],
indirect=True,
),
]
def _validate_mp4(video_bytes: bytes, min_frames: int = 10) -> None:
"""Validate that the MP4 contains meaningful video and audio tracks."""
container = av.open(io.BytesIO(video_bytes))
v_streams = [s for s in container.streams if s.type == "video"]
assert len(v_streams) >= 1, "No video stream found in MP4"
a_streams = [s for s in container.streams if s.type == "audio"]
assert len(a_streams) >= 1, "No audio stream found in MP4"
v_stream = v_streams[0]
assert v_stream.width >= 1080, f"Unexpected video width: {v_stream.width}"
assert v_stream.height >= 1056, f"Unexpected video height: {v_stream.height}"
frame_count = 0
for frame in container.decode(video=0):
frame_count += 1
if frame_count >= min_frames:
break
assert frame_count >= min_frames, f"Video has only {frame_count} frames (expected >= {min_frames})"
container.close()
@hardware_test(res={"cuda": "H100"}, num_cards=2)
def test_magi_human_e2e(omni_runner: OmniRunner):
"""End-to-end test for MagiHuman generating video and audio."""
prompt = (
"A young woman with long, wavy golden blonde hair and bright blue eyes, "
"wearing a fitted ivory silk blouse with a delicate lace collar, sits "
"stationary in front of a softly lit, blurred warm-toned interior. Her "
"overall disposition is warm, composed, and gently confident. The camera "
"holds a static medium close-up, framing her from the shoulders up, "
"with shallow depth of field keeping her face in sharp focus. Soft "
"directional key light falls from the upper left, casting a gentle "
"highlight along her cheekbone and nose bridge. She draws a quiet breath, "
"the levator labii superiors relaxing as her lips part. She speaks in "
"clear, warm, unhurried American English: "
"\"The most beautiful things in life aren't things at all — "
"they're moments, feelings, and the people who make you feel truly alive.\" "
"Her jaw descends smoothly on each stressed syllable; the orbicularis oris "
"shapes each vowel with precision. A faint, genuine smile engages the "
"zygomaticus major, lifting her lip corners fractionally. Her brows rest "
"in a soft, neutral arch throughout. She maintains steady, forward-facing "
"eye contact. Head position remains level; no torso displacement occurs.\n\n"
"Dialogue:\n"
"<Young blonde woman, American English>: "
"\"The most beautiful things in life aren't things at all — "
"they're moments, feelings, and the people who make you feel truly alive.\"\n\n"
"Background Sound:\n"
"<Soft, warm indoor ambience with a faint distant piano melody>"
)
sampling_params = OmniDiffusionSamplingParams(
height=256,
width=448,
num_inference_steps=8,
seed=52,
extra_args={
"seconds": 5,
"sr_height": 1080,
"sr_width": 1920,
"sr_num_inference_steps": 5,
},
)
omni = omni_runner.omni
outputs = list(
omni.generate(
prompts=[prompt],
sampling_params_list=[sampling_params],
)
)
assert len(outputs) > 0, "No outputs returned"
first = outputs[0]
assert hasattr(first, "images") and first.images, "No video frames in output"
video_frames = first.images[0]
assert isinstance(video_frames, np.ndarray), f"Expected numpy array, got {type(video_frames)}"
assert video_frames.ndim == 4, f"Expected 4D array (T,H,W,3), got shape {video_frames.shape}"
mm = first.multimodal_output
assert mm, "multimodal_output is empty or missing"
audio_waveform = mm.get("audio")
assert audio_waveform is not None, "No audio waveform in multimodal_output"
audio_sample_rate = mm.get("audio_sample_rate")
assert audio_sample_rate is not None, (
"audio_sample_rate not found in multimodal_output; model post-process must propagate it"
)
assert isinstance(audio_sample_rate, (int, float)), (
f"audio_sample_rate should be numeric, got {type(audio_sample_rate)}"
)
assert int(audio_sample_rate) > 0, f"audio_sample_rate must be positive, got {audio_sample_rate}"
fps = mm.get("fps")
assert fps is not None, "fps not found in multimodal_output; model post-process must propagate it"
assert isinstance(fps, (int, float)), f"fps should be numeric, got {type(fps)}"
assert int(fps) > 0, f"fps must be positive, got {fps}"
video_bytes = mux_video_audio_bytes(
video_frames,
audio_waveform,
fps=float(fps),
audio_sample_rate=int(audio_sample_rate),
)
assert isinstance(video_bytes, bytes), f"Expected MP4 bytes, got {type(video_bytes)}"
assert len(video_bytes) > 1000, f"MP4 too small ({len(video_bytes)} bytes)"
_validate_mp4(video_bytes)
tests/model_extras/test_model_extras.py
-17
View File
@@ -177,23 +177,6 @@ def test_cosmos3_extra_registry_declares_request_and_response_params(pipeline_na
assert should_init_extra_args_for_non_diffusion_stages(pipeline_name) is False
@pytest.mark.core_model
@pytest.mark.cpu
def test_magi_human_extra_registry_declares_request_and_response_params() -> None:
assert get_extra_body_params("MagiHumanPipeline") == frozenset(
{
"seconds",
"audio_path",
"image_path",
"sr_height",
"sr_width",
"sr_num_inference_steps",
}
)
assert get_extra_output_params("MagiHumanPipeline") == frozenset()
assert should_init_extra_args_for_non_diffusion_stages("MagiHumanPipeline") is False
@pytest.mark.core_model
@pytest.mark.cpu
def test_ltx_extra_registry_declares_official_guidance_params() -> None:
tests/model_tests/diffusion/test_alignment.py
-2
View File
@@ -57,7 +57,6 @@ EXCLUDED_MODELS = [
"OmniGen2Pipeline",
"HeliosPipeline",
"HeliosPyramidPipeline",
"DreamIDOmniPipeline",
"SenseNovaU1Pipeline",
"HunyuanVideo15Pipeline",
"HunyuanVideo15ImageToVideoPipeline",
@@ -71,7 +70,6 @@ EXCLUDED_MODELS = [
# Requires camera/action assets and persistent AR state; covered by its
# dedicated contract tests and real-checkpoint E2E smoke instead.
"LingBotWorldCausalDMDPipeline",
"MagiHumanPipeline",
"OmniVoicePipeline",
"OmniVoice",
"Cosmos3OmniDiffusersPipeline",