Merge branch 'main' into dev/vllm-align

This commit is contained in:
Gao Han authored and GitHub committed 2026-09-10 11:23:08 +08:00
commit 0ae9bdd8c2
54 files changed
+1659 -500

No files matched your search

@@ -26,6 +26,7 @@ _MODULE_NAME = "vllm_omni.benchmarks.data_modules.seed_tts_dataset"
if _MODULE_NAME not in sys.modules:
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
assert _spec is not None and _spec.loader is not None
_mod = importlib.util.module_from_spec(_spec)
sys.modules[_MODULE_NAME] = _mod
_spec.loader.exec_module(_mod)
@@ -237,7 +238,8 @@ def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_roo
assert req.seed_tts_utterance_id == "ok"
def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
@pytest.mark.parametrize("audio_seconds", [0.1, 61])
def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch, audio_seconds):
from vllm_omni.benchmarks.data_modules import seed_tts_eval
calls = {}
@@ -252,8 +254,9 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
return self
class FakeProcessor:
def __call__(self, wav, *, sampling_rate, return_tensors, return_attention_mask=False):
def __call__(self, wav, *, sampling_rate, return_tensors, return_attention_mask=False, **kwargs):
calls["return_attention_mask"] = return_attention_mask
calls["processor_kwargs"] = kwargs
assert sampling_rate == 16000
assert return_tensors == "pt"
assert len(wav) > 0
@@ -262,9 +265,10 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
attention_mask=FakeTensor("mask") if return_attention_mask else None,
)
def get_decoder_prompt_ids(self, *, language, task):
def get_decoder_prompt_ids(self, *, language, task, no_timestamps=True):
assert language == "english"
assert task == "transcribe"
calls["no_timestamps"] = no_timestamps
return [(1, 2)]
def batch_decode(self, predicted_ids, *, skip_special_tokens):
@@ -283,10 +287,18 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
monkeypatch.setattr(seed_tts_eval, "_en_model", FakeModel())
monkeypatch.setattr(seed_tts_eval, "_device", "cuda:1")
text = seed_tts_eval._transcribe_en_f32_16k(np.ones(1600, dtype=np.float32))
text = seed_tts_eval._transcribe_en_f32_16k(np.ones(int(audio_seconds * 16000), dtype=np.float32))
assert text == "hello"
assert calls["return_attention_mask"] is True
assert calls["input_features"].device == "cuda:1"
assert calls["generate_kwargs"]["attention_mask"].device == "cuda:1"
assert calls["generate_kwargs"]["forced_decoder_ids"] == [(1, 2)]
if audio_seconds > 30:
assert calls["processor_kwargs"] == {"truncation": False, "padding": "longest"}
assert calls["generate_kwargs"]["return_timestamps"] is True
assert calls["no_timestamps"] is False
else:
assert calls["processor_kwargs"] == {}
assert "return_timestamps" not in calls["generate_kwargs"]
assert calls["no_timestamps"] is True
+12 -4
View File
@@ -5,7 +5,7 @@ import pytest
import torch
import torch.nn.functional as F
from tests.helpers.mark import hardware_test
from tests.helpers.mark import hardware_marks
from vllm_omni.diffusion.layers.activation import SiluAndMul
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
@@ -19,9 +19,17 @@ def test_silu_and_mul_native_matches_packed_reference() -> None:
torch.testing.assert_close(SiluAndMul().forward_native(packed), F.silu(gate) * up)
@hardware_test(res={"npu": "A3"}, num_cards=1)
def test_silu_and_mul_npu_matches_packed_reference() -> None:
packed = torch.randn(257, 256, device="npu", dtype=torch.bfloat16)
@pytest.mark.parametrize(
"dtype",
[
pytest.param(torch.bfloat16, marks=hardware_marks(res={"npu": "A2"}, num_cards=1), id="A2-bf16"),
pytest.param(torch.bfloat16, marks=hardware_marks(res={"npu": "A3"}, num_cards=1), id="A3-bf16"),
pytest.param(torch.bfloat16, marks=hardware_marks(res={"npu": "A5"}, num_cards=1), id="A5-bf16"),
pytest.param(torch.float16, marks=hardware_marks(res={"npu": "310P"}, num_cards=1), id="310P-fp16"),
],
)
def test_silu_and_mul_npu_matches_packed_reference(dtype: torch.dtype) -> None:
packed = torch.randn(257, 256, device="npu", dtype=dtype)
gate, up = packed.chunk(2, dim=-1)
torch.testing.assert_close(
@@ -37,8 +37,10 @@ def clear_platform_cache():
(90, True, True, True, True),
(90, True, False, True, False),
(90, False, True, False, False),
(100, True, True, False, False),
(103, True, True, False, False),
(100, True, True, True, False),
(100, True, False, True, False),
(103, True, True, True, False),
(103, True, False, True, False),
(110, True, True, False, False),
(120, True, True, False, False),
(121, True, True, False, False),
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CUDA regression tests for LTX-2 BWE vocoder precision."""
"""GPU regression tests for LTX-2 BWE vocoder precision."""
import pytest
import torch
@@ -14,10 +14,13 @@ pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
class _BWEConvVocoder(torch.nn.Module):
def __init__(self):
super().__init__()
# Match the production LTX vocoder's conv_pre geometry. The previous
# single-channel 1x1 probe intermittently had no MIOpen algorithm.
self.conv = torch.nn.Conv1d(
1,
1,
kernel_size=1,
128,
1024,
kernel_size=7,
padding=3,
bias=False,
device="cuda",
dtype=torch.bfloat16,
@@ -33,12 +36,12 @@ class _BWEConvVocoder(torch.nn.Module):
return output
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
def test_ltx_bwe_vocoder_runs_real_cuda_autocast_in_fp32():
from vllm_omni.diffusion.models.ltx2.ltx2_runtime import _run_ltx_vocoder
vocoder = _BWEConvVocoder()
generated_mel = torch.ones((1, 1, 4), device="cuda", dtype=torch.bfloat16)
generated_mel = torch.ones((1, 128, 64), device="cuda", dtype=torch.bfloat16)
output = _run_ltx_vocoder(vocoder, generated_mel)
@@ -300,7 +300,8 @@ def test_single_stage_zimage_fp8_uses_less_memory():
assert mem_fp8 < mem_bf16, f"FP8 ({mem_fp8:.2f} GiB) should use less memory than BF16 ({mem_bf16:.2f} GiB)"
@hardware_test(res={"cuda": "L4"})
@hardware_test(res={"cuda": "L4", "xpu": "B60"})
@pytest.mark.advanced_model
def test_single_stage_qwen_image_fp8():
"""Qwen-Image (random weights) with FP8 generates valid images."""
model = "riverclouds/qwen_image_random"
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Check replica startup ordering without model-loading timing assumptions."""
import contextlib
import fcntl
import os
import threading
from collections.abc import Iterator
from pathlib import Path
import pytest
from pytest_mock import MockerFixture
from vllm.config import VllmConfig
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.utils import CoreEngineProcManager, EngineZmqAddresses
from vllm_omni.config.stage_config import StageConfig
from vllm_omni.engine import stage_init_utils, stage_runtime
from vllm_omni.engine.stage_engine_startup import StageReplicaResources
from vllm_omni.engine.stage_init_utils import LogicalStageInitPlan, ReplicaInitPlan, StageMetadata
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def launch_runtime(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, mocker: MockerFixture
) -> Iterator[stage_runtime.StageRuntime]:
runtime = stage_runtime.StageRuntime(
stage_configs=[],
model="dummy-model",
config_path="dummy-config",
stage_init_timeout=5,
async_chunk=False,
)
# These are virtual device IDs: only the real per-device file locking and
# scheduling run. No model, CUDA context, or engine process is created.
monkeypatch.setattr(runtime, "_resolve_replica_physical_devices", lambda _stage, cfg: cfg["devices"])
monkeypatch.setattr(
stage_init_utils, "device_init_lock_path", lambda device: str(tmp_path / f"device-{device}.lock")
)
mocker.patch.object(
stage_runtime.StageEngineCoreClientBase,
"make_async_mp_client",
return_value=mocker.Mock(spec=stage_runtime.StageClient),
)
monkeypatch.setenv("VLLM_OMNI_TEST_LAUNCH_ENV", "parent")
yield runtime
if runtime._stage_init_executor is not None:
runtime._stage_init_executor.shutdown(wait=True)
def _plan(stage_id: int, device: str) -> LogicalStageInitPlan:
metadata = StageMetadata(
stage_id=stage_id,
stage_type="llm",
engine_output_type="text",
is_comprehension=False,
requires_multimodal_data=False,
engine_input_source=[],
final_output=False,
final_output_type=None,
default_sampling_params=SamplingParams(),
custom_process_input_func=None,
model_stage="dummy",
runtime_cfg={"devices": device, "env": {"VLLM_OMNI_TEST_LAUNCH_ENV": str(stage_id)}},
)
replica = ReplicaInitPlan(
replica_id=0,
num_replicas=1,
launch_mode="local",
stage_cfg=StageConfig(stage_id=stage_id, model_stage="dummy", yaml_runtime=metadata.runtime_cfg),
metadata=metadata,
stage_connector_spec={},
omni_kv_connector=(None, None, None),
stage_vllm_config=VllmConfig(),
executor_class=object,
engine_args_dict={},
)
return LogicalStageInitPlan(stage_idx=stage_id, stage_id=stage_id, replicas=[replica])
def _resources(mocker: MockerFixture) -> StageReplicaResources:
return StageReplicaResources(
manager=mocker.Mock(spec=CoreEngineProcManager),
addresses=EngineZmqAddresses(inputs=["unused-input"], outputs=["unused-output"]),
)
def test_distinct_devices_spawn_before_either_is_ready(
launch_runtime: stage_runtime.StageRuntime, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
) -> None:
spawned = [threading.Event(), threading.Event()]
@contextlib.contextmanager
def launch(*, stage_id: int, stage_visible_devices: str, **_kwargs: object) -> Iterator[StageReplicaResources]:
assert os.environ["VLLM_OMNI_TEST_LAUNCH_ENV"] == str(stage_id)
spawned[stage_id].set()
yield _resources(mocker)
# The device remains exclusively locked throughout loading/profiling,
# even though another device's replica must be able to start.
with open(stage_init_utils.device_init_lock_path(int(stage_visible_devices)), "r+") as lock:
with pytest.raises(BlockingIOError):
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
assert spawned[1 - stage_id].wait(5), "A different device's spawn was blocked by the READY wait"
monkeypatch.setattr(stage_runtime, "launch_stage_replica", launch)
clients = launch_runtime._initialize_stage_replicas([_plan(0, "0"), _plan(1, "1")], 5)
assert all(clients[stage][0] is not None for stage in (0, 1))
assert os.environ["VLLM_OMNI_TEST_LAUNCH_ENV"] == "parent"
def test_same_device_waits_for_ready_before_next_spawn(
launch_runtime: stage_runtime.StageRuntime, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
) -> None:
events: list[tuple[int, str]] = []
@contextlib.contextmanager
def launch(*, stage_id: int, **_kwargs: object) -> Iterator[StageReplicaResources]:
events.append((stage_id, "spawn"))
yield _resources(mocker)
events.append((stage_id, "ready"))
monkeypatch.setattr(stage_runtime, "launch_stage_replica", launch)
launch_runtime._initialize_stage_replicas([_plan(0, "0"), _plan(1, "0")], 5)
assert events == [(0, "spawn"), (0, "ready"), (1, "spawn"), (1, "ready")]
assert os.environ["VLLM_OMNI_TEST_LAUNCH_ENV"] == "parent"
def test_ready_failure_cleans_up_resources_and_releases_device(
launch_runtime: stage_runtime.StageRuntime, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
) -> None:
manager = mocker.Mock(spec=CoreEngineProcManager)
resources = StageReplicaResources(manager=manager, addresses=EngineZmqAddresses(inputs=[], outputs=[]))
@contextlib.contextmanager
def launch(**_kwargs: object) -> Iterator[StageReplicaResources]:
yield resources
raise RuntimeError("READY handshake failed")
monkeypatch.setattr(stage_runtime, "launch_stage_replica", launch)
with pytest.raises(RuntimeError, match="READY handshake failed"):
launch_runtime._initialize_local_llm_replica(_plan(0, "0").replicas[0], 5)
manager.shutdown.assert_called_once_with()
assert os.environ["VLLM_OMNI_TEST_LAUNCH_ENV"] == "parent"
with open(stage_init_utils.device_init_lock_path(0), "r+") as lock:
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
@@ -166,6 +166,56 @@ def test_moss_tts_seed_falls_back_to_stage_default(adapter_cls, mocker):
assert prepared.tts_params["seed"] == [42]
@pytest.mark.parametrize(
"variant,ref_text,mode",
[
("local", " Reference. ", "continuation"),
("local", None, "generation"),
("local", " ", "generation"),
("tts", "Reference.", "generation"),
],
)
def test_moss_reference_transcript_mode(variant, ref_text, mode, mocker):
import torch
reference = [torch.ones((3, 12), dtype=torch.int64)]
unified = torch.arange(52, dtype=torch.int64).reshape(1, 4, 13)
processor = mocker.Mock(return_value={"input_ids": unified})
server = mocker.Mock(_moss_variant="local", uploaded_speakers={"speaker": {}})
server._voice_created_at.return_value = 123
request = OpenAICreateSpeechRequest(
input="Target.", ref_text=ref_text, language="English", voice="Speaker", seed=0, max_new_tokens=2048
)
adapter = MossTTSAdapter(SpeechServingContext(server=server))
adapter._moss_variant = variant
mocker.patch.object(adapter, "_get_moss_processor", return_value=processor)
mocker.patch.object(
adapter, "_encode_moss_references", new=mocker.AsyncMock(return_value=(reference, {0: "reference-key"}))
)
prepared = asyncio.run(adapter.build(request, [], has_inline_ref_audio=False))
if mode == "continuation":
processor.build_user_message.assert_called_once_with(text="Reference. Target.", language="English")
processor.build_assistant_message.assert_called_once_with(audio_codes_list=reference)
conversation = [processor.build_user_message.return_value, processor.build_assistant_message.return_value]
else:
processor.build_user_message.assert_called_once_with(text="Target.", language="English", reference=reference)
processor.build_assistant_message.assert_not_called()
conversation = [processor.build_user_message.return_value]
processor.assert_called_once_with(conversations=[conversation], mode=mode)
assert prepared.prompt["prompt_token_ids"] == unified[0, :, 0].tolist()
assert torch.equal(prepared.tts_params["codes"]["ref"], unified[0, :, 1:])
assert prepared.tts_params["max_new_frames"] == [2048]
assert prepared.tts_params["ref_audio_cache_key"] == "reference-key"
assert prepared.tts_params["voice_name"] == ["speaker"]
assert prepared.tts_params["voice_created_at"] == [123]
assert prepared.tts_params["seed"] == [0]
assert prepared.prompt["cache_salt"]
assert request.input == "Target."
def test_qwen3_tts_metadata():
assert Qwen3TTSAdapter.backend == "ar"
assert issubclass(Qwen3TTSAdapter, ARTTSAdapter)
+25 -1
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import queue
@@ -97,9 +100,11 @@ class FakeAsyncOmniEngine:
) -> None:
self.model = model
self.config_path = None
self.stage_configs: list[Any] = []
self.stage_metadata = stage_metadata or [THREE_STAGE_META[-1]]
self.num_stages = len(self.stage_metadata)
self.stage_configs = [
StageConfig(stage_id=i, model_stage="dummy-model").to_omegaconf() for i in range(self.num_stages)
]
self.default_sampling_params_list = default_sampling_params_list or [
SamplingParams(max_tokens=8) for _ in range(self.num_stages)
]
@@ -222,6 +227,25 @@ def test_resolve_sampling_params_list_preserves_stage_constraints():
assert caller_params.stop_token_ids == [7]
@pytest.mark.parametrize("use_defaults", [False, True])
def test_moss_local_output_policy_preserves_codec_streaming(use_defaults):
from vllm_omni.model_executor.models.moss_tts.pipeline import MOSS_TTS_LOCAL_PIPELINE
base = _make_base()
base.engine.num_stages = 2
base.sampling_constraints_list = [stage.sampling_constraints for stage in MOSS_TTS_LOCAL_PIPELINE.stages]
base.default_sampling_params_list = [
base._apply_sampling_constraints(SamplingParams(), constraints)
for constraints in base.sampling_constraints_list
]
caller = [SamplingParams(output_kind=RequestOutputKind.DELTA) for _ in range(2)]
result = base.resolve_sampling_params_list(None if use_defaults else caller, allow_delta_coercion=True)
assert [params.output_kind for params in result] == [RequestOutputKind.FINAL_ONLY, RequestOutputKind.DELTA]
assert all(params.output_kind == RequestOutputKind.DELTA for params in caller)
assert base.default_sampling_params_list[0].output_kind == RequestOutputKind.FINAL_ONLY
def _stage_spec(
stage_id: int,
*,
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU parity tests for MOSS-TTS Local Depth RoPE caching."""
import pytest
@@ -42,3 +42,16 @@ def test_rope_cache_matches_reference_and_reuses_storage() -> None:
# Smaller slices retain the fixed allocation used by CUDA Graph.
attn.prepare_rope_cache(6, device, torch.float32)
assert attn._rope_cos_cache.data_ptr() == cache_ptr
def test_frame_local_kv_matches_full_prefix_and_reuses_slots() -> None:
model = MossTTSLocalDepthTransformer(_test_config()).eval()
model.h[0].attn.prepare_rope_cache(12, torch.device("cpu"), torch.float32)
cache = (torch.empty(2, 4, 12, 8), torch.empty(2, 4, 12, 8))
with torch.inference_mode():
for _ in range(2):
inputs = torch.randn(2, 12, 32)
for position in range(12):
expected = model._forward_prefix(inputs[:, : position + 1])[:, -1:]
actual = model._forward_prefix(inputs[:, position : position + 1], cache, position)
torch.testing.assert_close(actual, expected, atol=1e-6, rtol=1e-5)
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import copy
import pytest
import torch
import torch.nn.functional as F
from vllm_omni.model_executor.models.moss_tts.audio_tokenizer_v2 import (
MossAudioTokenizerMultiheadAttention,
StreamingExecutionContext,
)
from vllm_omni.model_executor.models.moss_tts.streaming_attention import masked_attention
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
@pytest.mark.parametrize(
"batch,heads,queries,keys", [(2, 20, 1, 125), (4, 20, 15, 125), (1, 12, 480, 400), (4, 12, 32, 250)]
)
def test_masked_attention_matches_sdpa_with_strides_and_invalid_rows(batch, heads, queries, keys):
torch.manual_seed(7)
q = torch.randn(batch, queries, 3, heads, 64, device="cuda", dtype=torch.bfloat16)[:, :, 0].permute(0, 2, 1, 3)
k = torch.randn(batch, heads, keys, 64, device="cuda", dtype=torch.bfloat16)
v = torch.randn_like(k)
positions = torch.arange(keys, device="cuda").roll(17)
query_positions = torch.arange(queries, device="cuda") + keys - queries
mask = (positions[None, :] <= query_positions[:, None])[None, None].expand(batch, 1, -1, -1).clone()
mask[0, :, 0] = False # Cover an all-masked row alongside valid rows.
actual = masked_attention(q, k, v, mask)
expected = F.scaled_dot_product_attention(q, k, v, mask)
assert bool(actual.isfinite().all())
torch.testing.assert_close(actual, expected, atol=0.008, rtol=0.01)
assert actual[0, :, 0].count_nonzero() == 0
def test_streaming_attention_preserves_ring_wrap_reordering_and_slot_reset():
torch.manual_seed(9)
reference = MossAudioTokenizerMultiheadAttention(
128, 2, causal=True, context=9, device="cuda", dtype=torch.bfloat16
)
candidate = copy.deepcopy(reference)
candidate._streaming_attention = masked_attention
for model in (reference, candidate):
model._streaming_state = model._init_streaming_state(4)
for step in range(8):
slots = torch.tensor([1, 0, 3] if step % 2 else [0, 1, 3], device="cuda")
valid = torch.tensor([True, True, False], device="cuda")
context = StreamingExecutionContext(state_slot_ids=slots, valid_rows=valid)
x = torch.randn(3, 3, 128, device="cuda", dtype=torch.bfloat16)
with torch.inference_mode():
expected = reference(x, x, x, execution_context=context)
actual = candidate(x, x, x, execution_context=context)
torch.testing.assert_close(actual, expected, atol=0.01, rtol=0.02)
torch.testing.assert_close(candidate._streaming_state.offset, reference._streaming_state.offset, rtol=0, atol=0)
if step == 4:
for model in (reference, candidate):
model._streaming_state.reset_slots(torch.tensor([0], device="cuda"))
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_codec import (
MossTTSCodecDecoder,
_MossCodecStreamSession,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class CausalCodec(nn.Module):
downsample_rate = 1
def __init__(self):
super().__init__()
self.weight = nn.Parameter(torch.zeros(()))
def initialize_decoder_state_pool(self, capacity, scratch):
self.state = torch.zeros(capacity + scratch)
def reset_decoder_state_slots(self, slots):
self.state[slots] = 0
def decode_streaming_batch(self, codes, lengths, slots, valid_rows):
audio = codes.sum(0).float().cumsum(-1) + self.state[slots, None]
self.state[slots] = audio[:, -1]
return SimpleNamespace(audio=audio[:, None], audio_lengths=lengths)
def session():
return _MossCodecStreamSession(CausalCodec(), state_capacity=8, n_vq=2, vllm_config=None)
def test_terminal_padding_preserves_audio_live_state_and_reused_slots():
s = session()
assert [s.acquire(), s.acquire()] == [0, 1]
plan = {0: torch.ones(2, 3), 1: torch.ones(2, 15)}
with pytest.raises(ValueError, match="Only terminal"):
s.step(plan)
assert s._codec.state.count_nonzero() == 0
s.step({0: torch.ones(2, 1), 1: torch.full((2, 1), 2)})
actual = s.step(plan, terminal_slots={0})
torch.testing.assert_close(actual[0], torch.tensor([[4.0, 6.0, 8.0]]))
torch.testing.assert_close(actual[1], torch.arange(6.0, 36.0, 2.0)[None])
s.release(0, state_already_reset=True)
assert s.acquire() == 0
resumed = s.step({0: torch.ones(2, 1), 1: torch.ones(2, 1)})
assert [resumed[i].item() for i in (0, 1)] == [2, 36]
@pytest.mark.parametrize("graph_capacity, expected_calls", [(None, 5), (4, 5), (8, 3)])
def test_only_existing_compatible_graph_buckets_are_coalesced(mocker, graph_capacity, expected_calls):
s = session()
s._cudagraph_wrapper = (
None
if graph_capacity is None
else SimpleNamespace(
batch_sizes=[1, graph_capacity],
_select_frame_size=lambda size, allow_padding: next((b for b in [1, 8, 15] if b >= size), None),
decode=lambda *args, **kwargs: None,
)
)
step = mocker.spy(s, "step")
decoder = object.__new__(MossTTSCodecDecoder)
nn.Module.__init__(decoder)
decoder._stream_max_step_frames = 15
decoder._stream_state_capacity = 8
decoder._stream_req_slots = {}
decoder._ensure_stream_session = lambda: s
lengths = [3, 7, 8, 1, 15]
finished = [True, True, False, True, False]
result = decoder._decode_streaming_batch(
[
(i, str(i), torch.ones(2, length), done)
for i, (length, done) in enumerate(zip(lengths, finished, strict=True))
]
)
assert step.call_count == expected_calls
assert [result[i].shape[-1] for i in range(5)] == lengths
if graph_capacity == 8:
assert list(step.call_args_list[0].args[0]) == [0, 1, 2]
assert step.call_args_list[0].kwargs["terminal_slots"] == {0, 1}
assert list(step.call_args_list[1].args[0]) == [3] # Preserve the exact one-frame graph.
+76
View File
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import MossTTSLocalTalkerForGeneration
from vllm_omni.worker.gpu_ar_model_runner import _snapshot_tensor_payload_to_cpu_async
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
def test_async_packed_snapshot_survives_source_reuse():
values = [
torch.arange(12, device="cuda").view(1, 12),
torch.empty((0, 12), device="cuda", dtype=torch.long),
torch.arange(24, device="cuda").view(2, 12),
]
expected = [v.cpu() for v in values]
snapshot = _snapshot_tensor_payload_to_cpu_async(
{"list": values, "tuple": tuple(values)}, copy_stream=torch.cuda.Stream(), pin_memory=True
)
for v in values:
v.fill_(-99)
snapshot.wait()
torch.testing.assert_close(snapshot.payload, {"list": expected, "tuple": tuple(expected)}, rtol=0, atol=0)
@pytest.mark.parametrize("offsets", [[0, 1], [1, 4]])
def test_mtp_preserves_rows_and_snapshot_without_cuda_sync(monkeypatch, offsets):
import vllm_omni.worker.gpu_model_runner as mod
from tests.worker.test_omni_gpu_model_runner import _make_runner, _noop_forward_context
monkeypatch.setattr(mod.current_omni_platform, "set_forward_context", _noop_forward_context)
runner = _make_runner(req_ids=("r1", "r2"), hidden_size=4)
runner.model.gpu_resident_buffer_keys = {("codes", "audio")}
for name in ("talker_mtp_input_ids", "talker_mtp_inputs_embeds", "last_talker_hidden", "text_step"):
buffer = getattr(runner, name)
buffer.gpu = buffer.gpu.to("cuda")
embeds = torch.ones((2, 4), device="cuda")
codes = torch.arange(2, device="cuda").view(2, 1)
runner.talker_mtp = lambda *args, **kwargs: (embeds, codes)
output = torch.full((5, 4), -10.0, device="cuda")
previous = torch.cuda.get_sync_debug_mode()
try:
torch.cuda.set_sync_debug_mode("error")
runner._talker_mtp_forward(["r1", "r2"], output, offsets)
finally:
torch.cuda.set_sync_debug_mode(previous)
expected = torch.full((5, 4), -10.0)
expected[offsets] = 1.0
torch.testing.assert_close(output.cpu(), expected, rtol=0, atol=0)
codes.fill_(999)
assert runner.model_intermediate_buffer["r1"]["codes"]["audio"].item() == 0
assert runner.model_intermediate_buffer["r2"]["codes"]["audio"].item() == 1
def test_mixed_stop_flags_do_not_synchronize_cuda():
model = object.__new__(MossTTSLocalTalkerForGeneration)
torch.nn.Module.__init__(model)
model.n_vq = 12
model.audio_pad_token_id = 1024
valid = torch.arange(12, device="cuda").view(1, 12)
stopped = torch.full_like(valid, 1024)
infos = [{"audio_codes": {"current": valid}}, {}, {"audio_codes": {"current": stopped}}]
hidden = torch.ones((3, 4), device="cuda")
previous = torch.cuda.get_sync_debug_mode()
try:
torch.cuda.set_sync_debug_mode("error")
output = model.make_omni_output(hidden, runtime_additional_information=infos)
finally:
torch.cuda.set_sync_debug_mode(previous)
assert model._batch_should_continue.tolist() == [True, True, False]
torch.testing.assert_close(output.multimodal_outputs["codes"]["audio"], [valid, valid[:0], stopped])
assert all("current" not in info.get("audio_codes", {}) for info in infos)
prefill = model.make_omni_output(hidden[:1], runtime_additional_information=[{}])
assert model._batch_should_continue is None and prefill.multimodal_outputs == {}