Merge branch 'main' into dev/vllm-align
This commit is contained in:
54 files changed
+1659
-500
No files matched your search
@@ -26,6 +26,7 @@ _MODULE_NAME = "vllm_omni.benchmarks.data_modules.seed_tts_dataset"
|
||||
|
||||
if _MODULE_NAME not in sys.modules:
|
||||
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
|
||||
assert _spec is not None and _spec.loader is not None
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_MODULE_NAME] = _mod
|
||||
_spec.loader.exec_module(_mod)
|
||||
@@ -237,7 +238,8 @@ def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_roo
|
||||
assert req.seed_tts_utterance_id == "ok"
|
||||
|
||||
|
||||
def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
|
||||
@pytest.mark.parametrize("audio_seconds", [0.1, 61])
|
||||
def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch, audio_seconds):
|
||||
from vllm_omni.benchmarks.data_modules import seed_tts_eval
|
||||
|
||||
calls = {}
|
||||
@@ -252,8 +254,9 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
|
||||
return self
|
||||
|
||||
class FakeProcessor:
|
||||
def __call__(self, wav, *, sampling_rate, return_tensors, return_attention_mask=False):
|
||||
def __call__(self, wav, *, sampling_rate, return_tensors, return_attention_mask=False, **kwargs):
|
||||
calls["return_attention_mask"] = return_attention_mask
|
||||
calls["processor_kwargs"] = kwargs
|
||||
assert sampling_rate == 16000
|
||||
assert return_tensors == "pt"
|
||||
assert len(wav) > 0
|
||||
@@ -262,9 +265,10 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
|
||||
attention_mask=FakeTensor("mask") if return_attention_mask else None,
|
||||
)
|
||||
|
||||
def get_decoder_prompt_ids(self, *, language, task):
|
||||
def get_decoder_prompt_ids(self, *, language, task, no_timestamps=True):
|
||||
assert language == "english"
|
||||
assert task == "transcribe"
|
||||
calls["no_timestamps"] = no_timestamps
|
||||
return [(1, 2)]
|
||||
|
||||
def batch_decode(self, predicted_ids, *, skip_special_tokens):
|
||||
@@ -283,10 +287,18 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
|
||||
monkeypatch.setattr(seed_tts_eval, "_en_model", FakeModel())
|
||||
monkeypatch.setattr(seed_tts_eval, "_device", "cuda:1")
|
||||
|
||||
text = seed_tts_eval._transcribe_en_f32_16k(np.ones(1600, dtype=np.float32))
|
||||
text = seed_tts_eval._transcribe_en_f32_16k(np.ones(int(audio_seconds * 16000), dtype=np.float32))
|
||||
|
||||
assert text == "hello"
|
||||
assert calls["return_attention_mask"] is True
|
||||
assert calls["input_features"].device == "cuda:1"
|
||||
assert calls["generate_kwargs"]["attention_mask"].device == "cuda:1"
|
||||
assert calls["generate_kwargs"]["forced_decoder_ids"] == [(1, 2)]
|
||||
if audio_seconds > 30:
|
||||
assert calls["processor_kwargs"] == {"truncation": False, "padding": "longest"}
|
||||
assert calls["generate_kwargs"]["return_timestamps"] is True
|
||||
assert calls["no_timestamps"] is False
|
||||
else:
|
||||
assert calls["processor_kwargs"] == {}
|
||||
assert "return_timestamps" not in calls["generate_kwargs"]
|
||||
assert calls["no_timestamps"] is True
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from vllm_omni.diffusion.layers.activation import SiluAndMul
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
|
||||
@@ -19,9 +19,17 @@ def test_silu_and_mul_native_matches_packed_reference() -> None:
|
||||
torch.testing.assert_close(SiluAndMul().forward_native(packed), F.silu(gate) * up)
|
||||
|
||||
|
||||
@hardware_test(res={"npu": "A3"}, num_cards=1)
|
||||
def test_silu_and_mul_npu_matches_packed_reference() -> None:
|
||||
packed = torch.randn(257, 256, device="npu", dtype=torch.bfloat16)
|
||||
@pytest.mark.parametrize(
|
||||
"dtype",
|
||||
[
|
||||
pytest.param(torch.bfloat16, marks=hardware_marks(res={"npu": "A2"}, num_cards=1), id="A2-bf16"),
|
||||
pytest.param(torch.bfloat16, marks=hardware_marks(res={"npu": "A3"}, num_cards=1), id="A3-bf16"),
|
||||
pytest.param(torch.bfloat16, marks=hardware_marks(res={"npu": "A5"}, num_cards=1), id="A5-bf16"),
|
||||
pytest.param(torch.float16, marks=hardware_marks(res={"npu": "310P"}, num_cards=1), id="310P-fp16"),
|
||||
],
|
||||
)
|
||||
def test_silu_and_mul_npu_matches_packed_reference(dtype: torch.dtype) -> None:
|
||||
packed = torch.randn(257, 256, device="npu", dtype=dtype)
|
||||
gate, up = packed.chunk(2, dim=-1)
|
||||
|
||||
torch.testing.assert_close(
|
||||
|
||||
@@ -37,8 +37,10 @@ def clear_platform_cache():
|
||||
(90, True, True, True, True),
|
||||
(90, True, False, True, False),
|
||||
(90, False, True, False, False),
|
||||
(100, True, True, False, False),
|
||||
(103, True, True, False, False),
|
||||
(100, True, True, True, False),
|
||||
(100, True, False, True, False),
|
||||
(103, True, True, True, False),
|
||||
(103, True, False, True, False),
|
||||
(110, True, True, False, False),
|
||||
(120, True, True, False, False),
|
||||
(121, True, True, False, False),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""CUDA regression tests for LTX-2 BWE vocoder precision."""
|
||||
"""GPU regression tests for LTX-2 BWE vocoder precision."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -14,10 +14,13 @@ pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
|
||||
class _BWEConvVocoder(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# Match the production LTX vocoder's conv_pre geometry. The previous
|
||||
# single-channel 1x1 probe intermittently had no MIOpen algorithm.
|
||||
self.conv = torch.nn.Conv1d(
|
||||
1,
|
||||
1,
|
||||
kernel_size=1,
|
||||
128,
|
||||
1024,
|
||||
kernel_size=7,
|
||||
padding=3,
|
||||
bias=False,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
@@ -33,12 +36,12 @@ class _BWEConvVocoder(torch.nn.Module):
|
||||
return output
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
def test_ltx_bwe_vocoder_runs_real_cuda_autocast_in_fp32():
|
||||
from vllm_omni.diffusion.models.ltx2.ltx2_runtime import _run_ltx_vocoder
|
||||
|
||||
vocoder = _BWEConvVocoder()
|
||||
generated_mel = torch.ones((1, 1, 4), device="cuda", dtype=torch.bfloat16)
|
||||
generated_mel = torch.ones((1, 128, 64), device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
output = _run_ltx_vocoder(vocoder, generated_mel)
|
||||
|
||||
|
||||
@@ -300,7 +300,8 @@ def test_single_stage_zimage_fp8_uses_less_memory():
|
||||
assert mem_fp8 < mem_bf16, f"FP8 ({mem_fp8:.2f} GiB) should use less memory than BF16 ({mem_bf16:.2f} GiB)"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
@hardware_test(res={"cuda": "L4", "xpu": "B60"})
|
||||
@pytest.mark.advanced_model
|
||||
def test_single_stage_qwen_image_fp8():
|
||||
"""Qwen-Image (random weights) with FP8 generates valid images."""
|
||||
model = "riverclouds/qwen_image_random"
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Check replica startup ordering without model-loading timing assumptions."""
|
||||
|
||||
import contextlib
|
||||
import fcntl
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.utils import CoreEngineProcManager, EngineZmqAddresses
|
||||
|
||||
from vllm_omni.config.stage_config import StageConfig
|
||||
from vllm_omni.engine import stage_init_utils, stage_runtime
|
||||
from vllm_omni.engine.stage_engine_startup import StageReplicaResources
|
||||
from vllm_omni.engine.stage_init_utils import LogicalStageInitPlan, ReplicaInitPlan, StageMetadata
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def launch_runtime(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, mocker: MockerFixture
|
||||
) -> Iterator[stage_runtime.StageRuntime]:
|
||||
runtime = stage_runtime.StageRuntime(
|
||||
stage_configs=[],
|
||||
model="dummy-model",
|
||||
config_path="dummy-config",
|
||||
stage_init_timeout=5,
|
||||
async_chunk=False,
|
||||
)
|
||||
# These are virtual device IDs: only the real per-device file locking and
|
||||
# scheduling run. No model, CUDA context, or engine process is created.
|
||||
monkeypatch.setattr(runtime, "_resolve_replica_physical_devices", lambda _stage, cfg: cfg["devices"])
|
||||
monkeypatch.setattr(
|
||||
stage_init_utils, "device_init_lock_path", lambda device: str(tmp_path / f"device-{device}.lock")
|
||||
)
|
||||
mocker.patch.object(
|
||||
stage_runtime.StageEngineCoreClientBase,
|
||||
"make_async_mp_client",
|
||||
return_value=mocker.Mock(spec=stage_runtime.StageClient),
|
||||
)
|
||||
monkeypatch.setenv("VLLM_OMNI_TEST_LAUNCH_ENV", "parent")
|
||||
yield runtime
|
||||
if runtime._stage_init_executor is not None:
|
||||
runtime._stage_init_executor.shutdown(wait=True)
|
||||
|
||||
|
||||
def _plan(stage_id: int, device: str) -> LogicalStageInitPlan:
|
||||
metadata = StageMetadata(
|
||||
stage_id=stage_id,
|
||||
stage_type="llm",
|
||||
engine_output_type="text",
|
||||
is_comprehension=False,
|
||||
requires_multimodal_data=False,
|
||||
engine_input_source=[],
|
||||
final_output=False,
|
||||
final_output_type=None,
|
||||
default_sampling_params=SamplingParams(),
|
||||
custom_process_input_func=None,
|
||||
model_stage="dummy",
|
||||
runtime_cfg={"devices": device, "env": {"VLLM_OMNI_TEST_LAUNCH_ENV": str(stage_id)}},
|
||||
)
|
||||
replica = ReplicaInitPlan(
|
||||
replica_id=0,
|
||||
num_replicas=1,
|
||||
launch_mode="local",
|
||||
stage_cfg=StageConfig(stage_id=stage_id, model_stage="dummy", yaml_runtime=metadata.runtime_cfg),
|
||||
metadata=metadata,
|
||||
stage_connector_spec={},
|
||||
omni_kv_connector=(None, None, None),
|
||||
stage_vllm_config=VllmConfig(),
|
||||
executor_class=object,
|
||||
engine_args_dict={},
|
||||
)
|
||||
return LogicalStageInitPlan(stage_idx=stage_id, stage_id=stage_id, replicas=[replica])
|
||||
|
||||
|
||||
def _resources(mocker: MockerFixture) -> StageReplicaResources:
|
||||
return StageReplicaResources(
|
||||
manager=mocker.Mock(spec=CoreEngineProcManager),
|
||||
addresses=EngineZmqAddresses(inputs=["unused-input"], outputs=["unused-output"]),
|
||||
)
|
||||
|
||||
|
||||
def test_distinct_devices_spawn_before_either_is_ready(
|
||||
launch_runtime: stage_runtime.StageRuntime, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
|
||||
) -> None:
|
||||
spawned = [threading.Event(), threading.Event()]
|
||||
|
||||
@contextlib.contextmanager
|
||||
def launch(*, stage_id: int, stage_visible_devices: str, **_kwargs: object) -> Iterator[StageReplicaResources]:
|
||||
assert os.environ["VLLM_OMNI_TEST_LAUNCH_ENV"] == str(stage_id)
|
||||
spawned[stage_id].set()
|
||||
yield _resources(mocker)
|
||||
# The device remains exclusively locked throughout loading/profiling,
|
||||
# even though another device's replica must be able to start.
|
||||
with open(stage_init_utils.device_init_lock_path(int(stage_visible_devices)), "r+") as lock:
|
||||
with pytest.raises(BlockingIOError):
|
||||
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
assert spawned[1 - stage_id].wait(5), "A different device's spawn was blocked by the READY wait"
|
||||
|
||||
monkeypatch.setattr(stage_runtime, "launch_stage_replica", launch)
|
||||
clients = launch_runtime._initialize_stage_replicas([_plan(0, "0"), _plan(1, "1")], 5)
|
||||
assert all(clients[stage][0] is not None for stage in (0, 1))
|
||||
assert os.environ["VLLM_OMNI_TEST_LAUNCH_ENV"] == "parent"
|
||||
|
||||
|
||||
def test_same_device_waits_for_ready_before_next_spawn(
|
||||
launch_runtime: stage_runtime.StageRuntime, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
|
||||
) -> None:
|
||||
events: list[tuple[int, str]] = []
|
||||
|
||||
@contextlib.contextmanager
|
||||
def launch(*, stage_id: int, **_kwargs: object) -> Iterator[StageReplicaResources]:
|
||||
events.append((stage_id, "spawn"))
|
||||
yield _resources(mocker)
|
||||
events.append((stage_id, "ready"))
|
||||
|
||||
monkeypatch.setattr(stage_runtime, "launch_stage_replica", launch)
|
||||
launch_runtime._initialize_stage_replicas([_plan(0, "0"), _plan(1, "0")], 5)
|
||||
assert events == [(0, "spawn"), (0, "ready"), (1, "spawn"), (1, "ready")]
|
||||
assert os.environ["VLLM_OMNI_TEST_LAUNCH_ENV"] == "parent"
|
||||
|
||||
|
||||
def test_ready_failure_cleans_up_resources_and_releases_device(
|
||||
launch_runtime: stage_runtime.StageRuntime, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
|
||||
) -> None:
|
||||
manager = mocker.Mock(spec=CoreEngineProcManager)
|
||||
resources = StageReplicaResources(manager=manager, addresses=EngineZmqAddresses(inputs=[], outputs=[]))
|
||||
|
||||
@contextlib.contextmanager
|
||||
def launch(**_kwargs: object) -> Iterator[StageReplicaResources]:
|
||||
yield resources
|
||||
raise RuntimeError("READY handshake failed")
|
||||
|
||||
monkeypatch.setattr(stage_runtime, "launch_stage_replica", launch)
|
||||
with pytest.raises(RuntimeError, match="READY handshake failed"):
|
||||
launch_runtime._initialize_local_llm_replica(_plan(0, "0").replicas[0], 5)
|
||||
|
||||
manager.shutdown.assert_called_once_with()
|
||||
assert os.environ["VLLM_OMNI_TEST_LAUNCH_ENV"] == "parent"
|
||||
with open(stage_init_utils.device_init_lock_path(0), "r+") as lock:
|
||||
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
@@ -166,6 +166,56 @@ def test_moss_tts_seed_falls_back_to_stage_default(adapter_cls, mocker):
|
||||
assert prepared.tts_params["seed"] == [42]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"variant,ref_text,mode",
|
||||
[
|
||||
("local", " Reference. ", "continuation"),
|
||||
("local", None, "generation"),
|
||||
("local", " ", "generation"),
|
||||
("tts", "Reference.", "generation"),
|
||||
],
|
||||
)
|
||||
def test_moss_reference_transcript_mode(variant, ref_text, mode, mocker):
|
||||
import torch
|
||||
|
||||
reference = [torch.ones((3, 12), dtype=torch.int64)]
|
||||
unified = torch.arange(52, dtype=torch.int64).reshape(1, 4, 13)
|
||||
processor = mocker.Mock(return_value={"input_ids": unified})
|
||||
server = mocker.Mock(_moss_variant="local", uploaded_speakers={"speaker": {}})
|
||||
server._voice_created_at.return_value = 123
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="Target.", ref_text=ref_text, language="English", voice="Speaker", seed=0, max_new_tokens=2048
|
||||
)
|
||||
adapter = MossTTSAdapter(SpeechServingContext(server=server))
|
||||
|
||||
adapter._moss_variant = variant
|
||||
mocker.patch.object(adapter, "_get_moss_processor", return_value=processor)
|
||||
mocker.patch.object(
|
||||
adapter, "_encode_moss_references", new=mocker.AsyncMock(return_value=(reference, {0: "reference-key"}))
|
||||
)
|
||||
|
||||
prepared = asyncio.run(adapter.build(request, [], has_inline_ref_audio=False))
|
||||
|
||||
if mode == "continuation":
|
||||
processor.build_user_message.assert_called_once_with(text="Reference. Target.", language="English")
|
||||
processor.build_assistant_message.assert_called_once_with(audio_codes_list=reference)
|
||||
conversation = [processor.build_user_message.return_value, processor.build_assistant_message.return_value]
|
||||
else:
|
||||
processor.build_user_message.assert_called_once_with(text="Target.", language="English", reference=reference)
|
||||
processor.build_assistant_message.assert_not_called()
|
||||
conversation = [processor.build_user_message.return_value]
|
||||
processor.assert_called_once_with(conversations=[conversation], mode=mode)
|
||||
assert prepared.prompt["prompt_token_ids"] == unified[0, :, 0].tolist()
|
||||
assert torch.equal(prepared.tts_params["codes"]["ref"], unified[0, :, 1:])
|
||||
assert prepared.tts_params["max_new_frames"] == [2048]
|
||||
assert prepared.tts_params["ref_audio_cache_key"] == "reference-key"
|
||||
assert prepared.tts_params["voice_name"] == ["speaker"]
|
||||
assert prepared.tts_params["voice_created_at"] == [123]
|
||||
assert prepared.tts_params["seed"] == [0]
|
||||
assert prepared.prompt["cache_salt"]
|
||||
assert request.input == "Target."
|
||||
|
||||
|
||||
def test_qwen3_tts_metadata():
|
||||
assert Qwen3TTSAdapter.backend == "ar"
|
||||
assert issubclass(Qwen3TTSAdapter, ARTTSAdapter)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
@@ -97,9 +100,11 @@ class FakeAsyncOmniEngine:
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.config_path = None
|
||||
self.stage_configs: list[Any] = []
|
||||
self.stage_metadata = stage_metadata or [THREE_STAGE_META[-1]]
|
||||
self.num_stages = len(self.stage_metadata)
|
||||
self.stage_configs = [
|
||||
StageConfig(stage_id=i, model_stage="dummy-model").to_omegaconf() for i in range(self.num_stages)
|
||||
]
|
||||
self.default_sampling_params_list = default_sampling_params_list or [
|
||||
SamplingParams(max_tokens=8) for _ in range(self.num_stages)
|
||||
]
|
||||
@@ -222,6 +227,25 @@ def test_resolve_sampling_params_list_preserves_stage_constraints():
|
||||
assert caller_params.stop_token_ids == [7]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_defaults", [False, True])
|
||||
def test_moss_local_output_policy_preserves_codec_streaming(use_defaults):
|
||||
from vllm_omni.model_executor.models.moss_tts.pipeline import MOSS_TTS_LOCAL_PIPELINE
|
||||
|
||||
base = _make_base()
|
||||
base.engine.num_stages = 2
|
||||
base.sampling_constraints_list = [stage.sampling_constraints for stage in MOSS_TTS_LOCAL_PIPELINE.stages]
|
||||
base.default_sampling_params_list = [
|
||||
base._apply_sampling_constraints(SamplingParams(), constraints)
|
||||
for constraints in base.sampling_constraints_list
|
||||
]
|
||||
caller = [SamplingParams(output_kind=RequestOutputKind.DELTA) for _ in range(2)]
|
||||
result = base.resolve_sampling_params_list(None if use_defaults else caller, allow_delta_coercion=True)
|
||||
|
||||
assert [params.output_kind for params in result] == [RequestOutputKind.FINAL_ONLY, RequestOutputKind.DELTA]
|
||||
assert all(params.output_kind == RequestOutputKind.DELTA for params in caller)
|
||||
assert base.default_sampling_params_list[0].output_kind == RequestOutputKind.FINAL_ONLY
|
||||
|
||||
|
||||
def _stage_spec(
|
||||
stage_id: int,
|
||||
*,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU parity tests for MOSS-TTS Local Depth RoPE caching."""
|
||||
|
||||
import pytest
|
||||
@@ -42,3 +42,16 @@ def test_rope_cache_matches_reference_and_reuses_storage() -> None:
|
||||
# Smaller slices retain the fixed allocation used by CUDA Graph.
|
||||
attn.prepare_rope_cache(6, device, torch.float32)
|
||||
assert attn._rope_cos_cache.data_ptr() == cache_ptr
|
||||
|
||||
|
||||
def test_frame_local_kv_matches_full_prefix_and_reuses_slots() -> None:
|
||||
model = MossTTSLocalDepthTransformer(_test_config()).eval()
|
||||
model.h[0].attn.prepare_rope_cache(12, torch.device("cpu"), torch.float32)
|
||||
cache = (torch.empty(2, 4, 12, 8), torch.empty(2, 4, 12, 8))
|
||||
with torch.inference_mode():
|
||||
for _ in range(2):
|
||||
inputs = torch.randn(2, 12, 32)
|
||||
for position in range(12):
|
||||
expected = model._forward_prefix(inputs[:, : position + 1])[:, -1:]
|
||||
actual = model._forward_prefix(inputs[:, position : position + 1], cache, position)
|
||||
torch.testing.assert_close(actual, expected, atol=1e-6, rtol=1e-5)
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm_omni.model_executor.models.moss_tts.audio_tokenizer_v2 import (
|
||||
MossAudioTokenizerMultiheadAttention,
|
||||
StreamingExecutionContext,
|
||||
)
|
||||
from vllm_omni.model_executor.models.moss_tts.streaming_attention import masked_attention
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch,heads,queries,keys", [(2, 20, 1, 125), (4, 20, 15, 125), (1, 12, 480, 400), (4, 12, 32, 250)]
|
||||
)
|
||||
def test_masked_attention_matches_sdpa_with_strides_and_invalid_rows(batch, heads, queries, keys):
|
||||
torch.manual_seed(7)
|
||||
q = torch.randn(batch, queries, 3, heads, 64, device="cuda", dtype=torch.bfloat16)[:, :, 0].permute(0, 2, 1, 3)
|
||||
k = torch.randn(batch, heads, keys, 64, device="cuda", dtype=torch.bfloat16)
|
||||
v = torch.randn_like(k)
|
||||
positions = torch.arange(keys, device="cuda").roll(17)
|
||||
query_positions = torch.arange(queries, device="cuda") + keys - queries
|
||||
mask = (positions[None, :] <= query_positions[:, None])[None, None].expand(batch, 1, -1, -1).clone()
|
||||
mask[0, :, 0] = False # Cover an all-masked row alongside valid rows.
|
||||
actual = masked_attention(q, k, v, mask)
|
||||
expected = F.scaled_dot_product_attention(q, k, v, mask)
|
||||
assert bool(actual.isfinite().all())
|
||||
torch.testing.assert_close(actual, expected, atol=0.008, rtol=0.01)
|
||||
assert actual[0, :, 0].count_nonzero() == 0
|
||||
|
||||
|
||||
def test_streaming_attention_preserves_ring_wrap_reordering_and_slot_reset():
|
||||
torch.manual_seed(9)
|
||||
reference = MossAudioTokenizerMultiheadAttention(
|
||||
128, 2, causal=True, context=9, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
candidate = copy.deepcopy(reference)
|
||||
candidate._streaming_attention = masked_attention
|
||||
for model in (reference, candidate):
|
||||
model._streaming_state = model._init_streaming_state(4)
|
||||
for step in range(8):
|
||||
slots = torch.tensor([1, 0, 3] if step % 2 else [0, 1, 3], device="cuda")
|
||||
valid = torch.tensor([True, True, False], device="cuda")
|
||||
context = StreamingExecutionContext(state_slot_ids=slots, valid_rows=valid)
|
||||
x = torch.randn(3, 3, 128, device="cuda", dtype=torch.bfloat16)
|
||||
with torch.inference_mode():
|
||||
expected = reference(x, x, x, execution_context=context)
|
||||
actual = candidate(x, x, x, execution_context=context)
|
||||
torch.testing.assert_close(actual, expected, atol=0.01, rtol=0.02)
|
||||
torch.testing.assert_close(candidate._streaming_state.offset, reference._streaming_state.offset, rtol=0, atol=0)
|
||||
if step == 4:
|
||||
for model in (reference, candidate):
|
||||
model._streaming_state.reset_slots(torch.tensor([0], device="cuda"))
|
||||
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_codec import (
|
||||
MossTTSCodecDecoder,
|
||||
_MossCodecStreamSession,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class CausalCodec(nn.Module):
|
||||
downsample_rate = 1
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(()))
|
||||
|
||||
def initialize_decoder_state_pool(self, capacity, scratch):
|
||||
self.state = torch.zeros(capacity + scratch)
|
||||
|
||||
def reset_decoder_state_slots(self, slots):
|
||||
self.state[slots] = 0
|
||||
|
||||
def decode_streaming_batch(self, codes, lengths, slots, valid_rows):
|
||||
audio = codes.sum(0).float().cumsum(-1) + self.state[slots, None]
|
||||
self.state[slots] = audio[:, -1]
|
||||
return SimpleNamespace(audio=audio[:, None], audio_lengths=lengths)
|
||||
|
||||
|
||||
def session():
|
||||
return _MossCodecStreamSession(CausalCodec(), state_capacity=8, n_vq=2, vllm_config=None)
|
||||
|
||||
|
||||
def test_terminal_padding_preserves_audio_live_state_and_reused_slots():
|
||||
s = session()
|
||||
assert [s.acquire(), s.acquire()] == [0, 1]
|
||||
plan = {0: torch.ones(2, 3), 1: torch.ones(2, 15)}
|
||||
with pytest.raises(ValueError, match="Only terminal"):
|
||||
s.step(plan)
|
||||
assert s._codec.state.count_nonzero() == 0
|
||||
s.step({0: torch.ones(2, 1), 1: torch.full((2, 1), 2)})
|
||||
actual = s.step(plan, terminal_slots={0})
|
||||
torch.testing.assert_close(actual[0], torch.tensor([[4.0, 6.0, 8.0]]))
|
||||
torch.testing.assert_close(actual[1], torch.arange(6.0, 36.0, 2.0)[None])
|
||||
s.release(0, state_already_reset=True)
|
||||
assert s.acquire() == 0
|
||||
resumed = s.step({0: torch.ones(2, 1), 1: torch.ones(2, 1)})
|
||||
assert [resumed[i].item() for i in (0, 1)] == [2, 36]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("graph_capacity, expected_calls", [(None, 5), (4, 5), (8, 3)])
|
||||
def test_only_existing_compatible_graph_buckets_are_coalesced(mocker, graph_capacity, expected_calls):
|
||||
s = session()
|
||||
s._cudagraph_wrapper = (
|
||||
None
|
||||
if graph_capacity is None
|
||||
else SimpleNamespace(
|
||||
batch_sizes=[1, graph_capacity],
|
||||
_select_frame_size=lambda size, allow_padding: next((b for b in [1, 8, 15] if b >= size), None),
|
||||
decode=lambda *args, **kwargs: None,
|
||||
)
|
||||
)
|
||||
step = mocker.spy(s, "step")
|
||||
decoder = object.__new__(MossTTSCodecDecoder)
|
||||
nn.Module.__init__(decoder)
|
||||
decoder._stream_max_step_frames = 15
|
||||
decoder._stream_state_capacity = 8
|
||||
decoder._stream_req_slots = {}
|
||||
decoder._ensure_stream_session = lambda: s
|
||||
lengths = [3, 7, 8, 1, 15]
|
||||
finished = [True, True, False, True, False]
|
||||
result = decoder._decode_streaming_batch(
|
||||
[
|
||||
(i, str(i), torch.ones(2, length), done)
|
||||
for i, (length, done) in enumerate(zip(lengths, finished, strict=True))
|
||||
]
|
||||
)
|
||||
assert step.call_count == expected_calls
|
||||
assert [result[i].shape[-1] for i in range(5)] == lengths
|
||||
if graph_capacity == 8:
|
||||
assert list(step.call_args_list[0].args[0]) == [0, 1, 2]
|
||||
assert step.call_args_list[0].kwargs["terminal_slots"] == {0, 1}
|
||||
assert list(step.call_args_list[1].args[0]) == [3] # Preserve the exact one-frame graph.
|
||||
@@ -0,0 +1,76 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import MossTTSLocalTalkerForGeneration
|
||||
from vllm_omni.worker.gpu_ar_model_runner import _snapshot_tensor_payload_to_cpu_async
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
|
||||
|
||||
|
||||
def test_async_packed_snapshot_survives_source_reuse():
|
||||
values = [
|
||||
torch.arange(12, device="cuda").view(1, 12),
|
||||
torch.empty((0, 12), device="cuda", dtype=torch.long),
|
||||
torch.arange(24, device="cuda").view(2, 12),
|
||||
]
|
||||
expected = [v.cpu() for v in values]
|
||||
snapshot = _snapshot_tensor_payload_to_cpu_async(
|
||||
{"list": values, "tuple": tuple(values)}, copy_stream=torch.cuda.Stream(), pin_memory=True
|
||||
)
|
||||
for v in values:
|
||||
v.fill_(-99)
|
||||
snapshot.wait()
|
||||
torch.testing.assert_close(snapshot.payload, {"list": expected, "tuple": tuple(expected)}, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offsets", [[0, 1], [1, 4]])
|
||||
def test_mtp_preserves_rows_and_snapshot_without_cuda_sync(monkeypatch, offsets):
|
||||
import vllm_omni.worker.gpu_model_runner as mod
|
||||
from tests.worker.test_omni_gpu_model_runner import _make_runner, _noop_forward_context
|
||||
|
||||
monkeypatch.setattr(mod.current_omni_platform, "set_forward_context", _noop_forward_context)
|
||||
runner = _make_runner(req_ids=("r1", "r2"), hidden_size=4)
|
||||
runner.model.gpu_resident_buffer_keys = {("codes", "audio")}
|
||||
for name in ("talker_mtp_input_ids", "talker_mtp_inputs_embeds", "last_talker_hidden", "text_step"):
|
||||
buffer = getattr(runner, name)
|
||||
buffer.gpu = buffer.gpu.to("cuda")
|
||||
embeds = torch.ones((2, 4), device="cuda")
|
||||
codes = torch.arange(2, device="cuda").view(2, 1)
|
||||
runner.talker_mtp = lambda *args, **kwargs: (embeds, codes)
|
||||
output = torch.full((5, 4), -10.0, device="cuda")
|
||||
previous = torch.cuda.get_sync_debug_mode()
|
||||
try:
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
runner._talker_mtp_forward(["r1", "r2"], output, offsets)
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(previous)
|
||||
expected = torch.full((5, 4), -10.0)
|
||||
expected[offsets] = 1.0
|
||||
torch.testing.assert_close(output.cpu(), expected, rtol=0, atol=0)
|
||||
codes.fill_(999)
|
||||
assert runner.model_intermediate_buffer["r1"]["codes"]["audio"].item() == 0
|
||||
assert runner.model_intermediate_buffer["r2"]["codes"]["audio"].item() == 1
|
||||
|
||||
|
||||
def test_mixed_stop_flags_do_not_synchronize_cuda():
|
||||
model = object.__new__(MossTTSLocalTalkerForGeneration)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.n_vq = 12
|
||||
model.audio_pad_token_id = 1024
|
||||
valid = torch.arange(12, device="cuda").view(1, 12)
|
||||
stopped = torch.full_like(valid, 1024)
|
||||
infos = [{"audio_codes": {"current": valid}}, {}, {"audio_codes": {"current": stopped}}]
|
||||
hidden = torch.ones((3, 4), device="cuda")
|
||||
previous = torch.cuda.get_sync_debug_mode()
|
||||
try:
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
output = model.make_omni_output(hidden, runtime_additional_information=infos)
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(previous)
|
||||
assert model._batch_should_continue.tolist() == [True, True, False]
|
||||
torch.testing.assert_close(output.multimodal_outputs["codes"]["audio"], [valid, valid[:0], stopped])
|
||||
assert all("current" not in info.get("audio_codes", {}) for info in infos)
|
||||
prefill = model.make_omni_output(hidden[:1], runtime_additional_information=[{}])
|
||||
assert model._batch_should_continue is None and prefill.multimodal_outputs == {}
|
||||
Reference in new issue
Block a user