[Model] Optimize Qwen3-TTS streaming throughput and first audio on CUDA (#8163)

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 authored and GitHub committed 2026-09-26 22:51:04 +08:00
1 parent 8a49c65bf9
commit e1303e5067
55 files changed
+4222 -230

No files matched your search

+1 -1
View File
@@ -166,7 +166,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
assert category_counts == {
EnvironmentVariableCategory.PUBLIC_OMNI: 30,
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 29,
EnvironmentVariableCategory.MODEL_SPECIFIC: 58,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
EnvironmentVariableCategory.INTERNAL: 2,
+51
View File
@@ -886,6 +886,7 @@ def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
def test_runtime_config_fields_match_structured_runtime_scope():
assert {f.name for f in fields(OmniStageRuntimeConfig)} == {
"cuda_mps",
"additional_config",
"distributed_executor_backend",
"worker_cls",
@@ -2051,6 +2052,28 @@ def test_async_chunk_rejects_mismatched_connector_edge(disabled_stage, builder):
builder(pipeline, user_deploy_config=deploy)
@pytest.mark.parametrize("scope", ["pipeline", "stage"])
@pytest.mark.parametrize("from_yaml", [False, True])
@pytest.mark.parametrize("builder", [merge_pipeline_deploy, VllmOmniConfig.from_pipeline_config])
def test_async_chunk_rejects_quoted_false_before_selecting_processors(tmp_path, scope, from_yaml, builder):
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
if from_yaml:
path = tmp_path / "quoted_false.yaml"
path.write_text(
'async_chunk: "false"\n' if scope == "pipeline" else 'stages:\n - stage_id: 0\n async_chunk: "false"\n'
)
deploy = load_deploy_config(path)
elif scope == "pipeline":
deploy = DeployConfig(async_chunk="false")
else:
deploy = DeployConfig(stages=[StageDeployConfig(stage_id=0, async_chunk="false")])
with pytest.raises(ValueError, match="async_chunk must be a boolean"):
if builder is merge_pipeline_deploy:
builder(pipeline, deploy)
else:
builder(pipeline, user_deploy_config=deploy)
@pytest.mark.parametrize("explicit", [False, True])
def test_diffusion_quantization_origin_survives_projection_and_transport(monkeypatch, explicit):
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
@@ -2169,3 +2192,31 @@ def test_structured_diffusion_stage_keeps_shared_globals_outside_diffusion():
def test_structured_diffusion_stage_rejects_explicit_shared_engine_field(field_name, config_kwargs):
with pytest.raises(ValueError, match=rf"stage 0.*{field_name}"):
_build_single_diffusion_config(**config_kwargs)
def test_mps_is_explicit_only_in_experimental_single_gpu_profile():
assert not load_deploy_config(_DEPLOY_DIR / "qwen3_tts.yaml").cuda_mps
deploy = load_deploy_config(_DEPLOY_DIR / "qwen3_tts_high_concurrency_mrv2_single_gpu.yaml")
assert deploy.cuda_mps
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
stages = merge_pipeline_deploy(pipeline, deploy)
assert all(stage.yaml_runtime["cuda_mps"] is True for stage in stages)
def test_mps_config_rejects_string_boolean(tmp_path):
path = tmp_path / "deploy.yaml"
path.write_text('cuda_mps: "false"\n')
with pytest.raises(ValueError, match="cuda_mps must be a boolean"):
load_deploy_config(path)
def test_mps_stays_in_runtime_instead_of_engine_arguments():
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
config = _from_pipeline_key(
"qwen3_tts", deploy_config_path=str(_DEPLOY_DIR / "qwen3_tts_high_concurrency_mrv2_single_gpu.yaml")
)
for stage in config.stage_configs:
assert stage.runtime_config.cuda_mps
args = build_engine_args_dict_from_omni_stage_config(stage, model="test-model")
assert "cuda_mps" not in args
@@ -66,6 +66,53 @@ default_tts_server_params = [
)
]
# Exercise the throughput path in the existing L4 lane with smaller capacities.
# The production H200 profile retains its own admission and graph buckets.
_FAST_PATH_STAGE_CONFIG = modify_stage_config(
get_deploy_config_path("qwen3_tts_high_concurrency_mrv2_single_gpu.yaml"),
updates={
# The temporary overlay lives outside deploy/, so anchor its parent.
"base_config": get_deploy_config_path("qwen3_tts_high_concurrency_mrv2.yaml"),
"cuda_mps": False,
"connectors.connector_of_shared_memory.extra.decode_batch_max_size": 2,
"connectors.connector_of_shared_memory.extra.decode_cudagraph_batch_sizes": [1, 2],
"stages": {
0: {
"max_num_seqs": 4,
"max_num_batched_tokens": 128,
"compilation_config.cudagraph_capture_sizes": [1, 2, 4, 8, 16, 32, 64, 128],
"compilation_config.max_cudagraph_capture_size": 128,
},
1: {"max_num_seqs": 4},
},
},
)
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize(
"omni_server",
[OmniServerParams(model=MODEL, stage_config_path=_FAST_PATH_STAGE_CONFIG, server_args=["--trust-remote-code"])],
indirect=True,
)
def test_cached_predictor_streaming_audio(omni_server, online_client) -> None:
"""Cover the optimized predictor, first audio and codec path with real weights at L3."""
online_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "The quick brown fox jumps over the lazy dog.",
"stream": True,
"stream_format": "audio",
"response_format": "wav",
"task_type": "CustomVoice",
"voice": "vivian",
},
request_num=4,
)
@pytest.mark.core_model
@pytest.mark.advanced_model
@@ -10,6 +10,7 @@ import threading
import time
import types
from dataclasses import dataclass, field
from subprocess import CompletedProcess
import pytest
from omegaconf import OmegaConf
@@ -70,6 +71,49 @@ def test_stage_runtime_env_accepts_typed_runtime_config(monkeypatch):
assert env_key not in os.environ
@pytest.mark.parametrize(
"invalid_key,invalid_value",
[("INVALID=ENV", "value"), ("INVALID\0ENV", "value"), ("INVALID_ENV", "value\0")],
ids=["equals-in-key", "nul-in-key", "nul-in-value"],
)
def test_stage_runtime_env_restores_partial_application(monkeypatch, invalid_key, invalid_value):
existing_key = "VLLM_OMNI_TEST_EXISTING_STAGE_ENV"
new_key = "VLLM_OMNI_TEST_NEW_STAGE_ENV"
monkeypatch.setenv(existing_key, "original")
monkeypatch.delenv(new_key, raising=False)
runtime_config = OmniStageRuntimeConfig(
env={existing_key: "overridden", new_key: "temporary", invalid_key: invalid_value}
)
with pytest.raises(ValueError):
with stage_runtime_env(0, runtime_config):
pytest.fail("Invalid environment must prevent stage launch")
assert os.environ[existing_key] == "original"
assert new_key not in os.environ
def test_stage_runtime_env_restores_after_launch_error(monkeypatch):
env_key = "VLLM_OMNI_TEST_STAGE_ENV_LAUNCH_ERROR"
monkeypatch.setenv(env_key, "original")
with pytest.raises(RuntimeError, match="stage launch failed"):
with stage_runtime_env(0, OmniStageRuntimeConfig(env={env_key: "temporary"})):
assert os.environ[env_key] == "temporary"
raise RuntimeError("stage launch failed")
assert os.environ[env_key] == "original"
def test_stage_runtime_env_restores_first_value_for_normalized_keys(monkeypatch):
monkeypatch.setenv("123", "original")
with stage_runtime_env(0, {"env": {123: "first", "123": "second"}}):
assert os.environ["123"] == "second"
assert os.environ["123"] == "original"
def test_orchestrator_startup_timeout_warns_how_to_raise_limits(monkeypatch):
engine = object.__new__(AsyncOmniEngine)
engine.orchestrator_thread = types.SimpleNamespace(is_alive=lambda: True)
@@ -1019,6 +1063,63 @@ def test_stage_runtime_launches_shared_engines_with_per_client_addresses(monkeyp
assert os.environ.get("VLLM_OMNI_TEST_STAGE_RUNTIME_ENV") is None
@pytest.mark.parametrize("client_count", [1, 2])
def test_stage_launch_uses_configured_mps_pipe_and_rejects_conflicts(monkeypatch, client_count):
import vllm_omni.engine.stage_runtime as runtime_mod
from vllm_omni.engine import cuda_mps
runtime = _make_stage_runtime()
plans = [_make_llm_plan(stage_id, stage_id=stage_id, vllm_config=_FakeVllmConfig()) for stage_id in (0, 1)]
for stage_id, plan in enumerate(plans):
plan.replicas[0].engine_args_dict = {}
plan.replicas[0].metadata.runtime_cfg = OmniStageRuntimeConfig(
cuda_mps=True,
devices="0",
env={"CUDA_MPS_PIPE_DIRECTORY": f"/operator/stage-{stage_id}"},
)
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/parent")
monkeypatch.setattr(runtime, "_prepare_stage_plans", lambda: plans)
monkeypatch.setattr(runtime, "_resolve_replica_physical_devices", lambda *_: "0")
monkeypatch.setattr(runtime_mod, "acquire_device_locks", lambda *_args, **_kwargs: [])
monkeypatch.setattr(runtime_mod, "physical_gpu_uuid", lambda _: "GPU-example")
monkeypatch.setattr(runtime_mod.current_omni_platform, "is_cuda", lambda: True)
monkeypatch.setattr(cuda_mps.shutil, "which", lambda _: "/bin/mps-control")
controls = []
def run_control(args, **kwargs):
controls.append(kwargs)
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
assert kwargs["env"]["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/stage-0"
return CompletedProcess(args, 0, stdout="")
monkeypatch.setattr(cuda_mps.subprocess, "run", run_control)
launched = []
@contextlib.contextmanager
def launch(**kwargs):
launched.append(kwargs["stage_id"])
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/stage-0"
assert kwargs["stage_visible_devices"] == "GPU-example"
yield StageReplicaResources(
addresses=EngineZmqAddresses(
inputs=[f"ipc://input-{i}" for i in range(client_count)],
outputs=[f"ipc://output-{i}" for i in range(client_count)],
)
)
monkeypatch.setattr(runtime_mod, "launch_stage_replica", launch)
try:
with pytest.raises(ValueError, match="Conflicting CUDA_MPS_PIPE_DIRECTORY"):
with runtime.launch_stage_engines(client_count):
pytest.fail("Conflicting colocated MPS policies must prevent launch")
assert launched == [0]
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
finally:
runtime.shutdown()
assert len(controls) == 1
assert controls[0]["input"] == "get_server_list\n"
@pytest.mark.parametrize("client_count", [1, 2])
def test_stage_runtime_overlapping_devices_acquire_real_locks_once(monkeypatch, tmp_path, client_count):
import fcntl
+200
View File
@@ -0,0 +1,200 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import os
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from vllm_omni.config.omni_config import OmniStageRuntimeConfig
from vllm_omni.engine import cuda_mps
from vllm_omni.engine.stage_runtime import StageRuntime
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def controls(monkeypatch):
monkeypatch.delenv("CUDA_MPS_PIPE_DIRECTORY", raising=False)
monkeypatch.setattr(cuda_mps.shutil, "which", lambda name: "/bin/mps-control")
run = Mock(return_value=SimpleNamespace(stdout=""))
monkeypatch.setattr(cuda_mps.subprocess, "run", run)
return run
def test_private_server_has_scoped_environment_and_idempotent_close(controls):
original = dict(os.environ)
server = cuda_mps.CudaMPSServer("GPU-example")
pipe = Path(server.env["CUDA_MPS_PIPE_DIRECTORY"])
assert pipe.is_dir()
assert controls.call_args_list[0].args[0] == ["/bin/mps-control", "-d"]
assert controls.call_args_list[0].kwargs["env"]["CUDA_VISIBLE_DEVICES"] == "GPU-example"
assert dict(os.environ) == original
server.close()
count = controls.call_count
server.close()
assert controls.call_count == count
assert controls.call_args.kwargs["input"] == "quit\n"
assert not pipe.exists()
def test_operator_daemon_is_checked_but_never_stopped(controls, monkeypatch):
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/mps")
server = cuda_mps.CudaMPSServer("GPU-example")
server.close()
assert controls.call_count == 1
assert controls.call_args.kwargs["input"] == "get_server_list\n"
def test_explicit_pipe_overrides_parent_without_taking_ownership(controls, monkeypatch):
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/parent")
server = cuda_mps.CudaMPSServer("GPU-example", pipe_directory="/operator/stage")
assert controls.call_args.kwargs["env"]["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/stage"
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
server.close()
assert controls.call_count == 1
def test_explicit_empty_pipe_selects_private_daemon(controls, monkeypatch):
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/parent")
server = cuda_mps.CudaMPSServer("GPU-example", pipe_directory="")
pipe = server.env["CUDA_MPS_PIPE_DIRECTORY"]
try:
assert pipe != "/operator/parent"
assert controls.call_args_list[0].args[0] == ["/bin/mps-control", "-d"]
assert controls.call_args_list[0].kwargs["env"]["CUDA_MPS_PIPE_DIRECTORY"] == pipe
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
finally:
server.close()
assert controls.call_args.kwargs["input"] == "quit\n"
assert controls.call_args.kwargs["env"]["CUDA_MPS_PIPE_DIRECTORY"] == pipe
assert not Path(pipe).exists()
@pytest.fixture
def mps_runtime(monkeypatch):
from vllm_omni.engine import stage_runtime
runtime = StageRuntime.__new__(StageRuntime)
runtime._mps_servers = {}
monkeypatch.setattr(stage_runtime, "physical_gpu_uuid", lambda device: "GPU-example")
monkeypatch.setattr(stage_runtime.current_omni_platform, "is_cuda", lambda: True)
yield runtime
runtime._close_mps_servers()
@pytest.mark.parametrize("pipe_directory", ["/operator/stage", ""])
def test_runtime_shares_first_stage_pipe_policy(controls, monkeypatch, mps_runtime, pipe_directory):
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/parent")
config = OmniStageRuntimeConfig(cuda_mps=True, env={"CUDA_MPS_PIPE_DIRECTORY": pipe_directory})
first_env = mps_runtime._mps_environment("0", config)
selected_pipe = first_env["CUDA_MPS_PIPE_DIRECTORY"]
if pipe_directory:
assert selected_pipe == pipe_directory
else:
assert selected_pipe != "/operator/parent"
assert Path(selected_pipe).is_dir()
call_count = controls.call_count
assert mps_runtime._mps_environment("0", {"cuda_mps": True}) is first_env
assert mps_runtime._mps_environment("0", config) is first_env
assert controls.call_count == call_count
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
@pytest.mark.parametrize(
"first_pipe,conflicting_pipe",
[("/operator/first", "/operator/other"), ("/operator/first", ""), ("", "/operator/other")],
)
def test_runtime_rejects_conflicting_explicit_stage_pipes(controls, mps_runtime, first_pipe, conflicting_pipe):
mps_runtime._mps_environment("0", {"cuda_mps": True, "env": {"CUDA_MPS_PIPE_DIRECTORY": first_pipe}})
call_count = controls.call_count
with pytest.raises(ValueError, match="Conflicting CUDA_MPS_PIPE_DIRECTORY"):
mps_runtime._mps_environment("0", {"cuda_mps": True, "env": {"CUDA_MPS_PIPE_DIRECTORY": conflicting_pipe}})
assert controls.call_count == call_count
def test_startup_failure_closes_only_the_private_daemon(controls):
controls.side_effect = [SimpleNamespace(stdout=""), RuntimeError("health failed"), SimpleNamespace(stdout="")]
with pytest.raises(RuntimeError, match="health failed"):
cuda_mps.CudaMPSServer("GPU-example")
assert controls.call_args.kwargs["input"] == "quit\n"
def test_runtime_opt_in_and_shared_gpu_ownership(monkeypatch):
from vllm_omni.engine import stage_runtime
runtime = StageRuntime.__new__(StageRuntime)
runtime._mps_servers = {}
create = Mock(return_value=SimpleNamespace(env={"CUDA_VISIBLE_DEVICES": "GPU-example"}, close=Mock()))
monkeypatch.setattr(stage_runtime, "CudaMPSServer", create)
monkeypatch.setattr(stage_runtime, "physical_gpu_uuid", lambda device: "GPU-example")
monkeypatch.setattr(stage_runtime, "current_omni_platform", SimpleNamespace(is_cuda=lambda: True))
assert runtime._mps_environment("0", {}) == {}
assert create.call_count == 0
for _ in range(2):
assert runtime._mps_environment("0", {"cuda_mps": True}) == {"CUDA_VISIBLE_DEVICES": "GPU-example"}
assert create.call_count == 1
runtime._close_mps_servers()
runtime._close_mps_servers()
create.return_value.close.assert_called_once()
@pytest.mark.parametrize("devices", [None, "0,1"])
def test_runtime_rejects_ambiguous_placement(devices, monkeypatch):
from vllm_omni.engine import stage_runtime
runtime = StageRuntime.__new__(StageRuntime)
monkeypatch.setattr(stage_runtime, "current_omni_platform", SimpleNamespace(is_cuda=lambda: True))
with pytest.raises(ValueError, match="exactly one explicit GPU"):
runtime._mps_environment(devices, {"cuda_mps": True})
def test_failed_shutdown_preserves_socket_for_retry(controls):
server = cuda_mps.CudaMPSServer("GPU-example")
pipe = Path(server.env["CUDA_MPS_PIPE_DIRECTORY"])
controls.side_effect = RuntimeError("quit failed")
with pytest.raises(RuntimeError, match="quit failed"):
server.close()
assert pipe.is_dir()
controls.side_effect = None
server.close()
assert not pipe.exists()
@pytest.mark.parametrize("launch_mode,stage_type", [("remote", "llm"), ("local", "diffusion")])
def test_mps_rejects_unsupported_stages(launch_mode, stage_type):
runtime = StageRuntime.__new__(StageRuntime)
metadata = SimpleNamespace(runtime_cfg={"cuda_mps": True}, stage_type=stage_type)
stage_cfg = SimpleNamespace(runtime=metadata.runtime_cfg)
plans = [
SimpleNamespace(replicas=[SimpleNamespace(metadata=metadata, stage_cfg=stage_cfg, launch_mode=launch_mode)])
]
with pytest.raises(ValueError, match="local EngineCore"):
runtime._validate_mps_topology(plans)
metadata.runtime_cfg["cuda_mps"] = False
runtime._validate_mps_topology(plans)
def test_mps_rejects_parallel_initialization():
runtime = StageRuntime.__new__(StageRuntime)
runtime._parallel_stage_init = True
replica = SimpleNamespace(
metadata=SimpleNamespace(stage_type="llm"),
stage_cfg=SimpleNamespace(runtime={"cuda_mps": True}),
launch_mode="local",
)
with pytest.raises(ValueError, match="parallel_stage_init=false"):
runtime._validate_mps_topology([SimpleNamespace(replicas=[replica])])
def test_failed_launch_command_still_checks_private_daemon_cleanup(controls):
controls.side_effect = [RuntimeError("launch failed after fork"), SimpleNamespace(stdout="")]
with pytest.raises(RuntimeError, match="launch failed after fork"):
cuda_mps.CudaMPSServer("GPU-example")
assert controls.call_args.kwargs["input"] == "quit\n"
+8
View File
@@ -425,3 +425,11 @@ class TestSerializeDeserializePayload:
restored = deserialize_payload(wire)
assert restored["meta"]["minimax_h3_prepared_reference_videos"] == descriptor
def test_first_audio_metadata_round_trips_as_a_tensor():
from vllm_omni.data_entry_keys import to_dict, to_struct
payload = {"meta": {"first_audio": torch.tensor(True)}}
restored = to_dict(to_struct(payload))
assert restored["meta"]["first_audio"].item() is True
@@ -0,0 +1,345 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
import torch
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine import FinishReason
from vllm_omni.engine import OmniEngineCoreOutput, OmniEngineCoreOutputs
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
from vllm_omni.engine.messages import ErrorMessage, OutputMessage
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _CodecPool:
final_output = True
def __init__(self, output_kind, registered, replica_id):
self.output_processor = MultimodalOutputProcessor(
tokenizer=None, log_stats=False, engine_core_output_type="audio"
)
self.output_kind = output_kind
self.replica_id = replica_id
self.bound_replica = None
if registered:
self.register()
def register(self):
self.bound_replica = self.replica_id
self.output_processor.request_states["r"] = OmniRequestState(
request_id="r",
external_req_id="r",
parent_req=None,
request_index=0,
lora_request=None,
prompt=None,
prompt_token_ids=[0],
prompt_embeds=None,
logprobs_processor=None,
detokenizer=None,
max_tokens_param=None,
arrival_time=0.0,
queue=None,
log_stats=False,
stream_interval=1,
output_kind=self.output_kind,
)
self.output_processor.external_req_ids["r"].append("r")
def get_bound_replica_id(self, request_id):
return self.bound_replica
async def process_llm_raw_outputs(self, replica_id, raw, **kwargs):
assert replica_id == self.replica_id
return self.output_processor.process_outputs(raw.outputs, raw.timestamp).request_outputs
def _orchestrator(output_kind=RequestOutputKind.DELTA, *, registered=True, replica_id=2, final_stage_id=1):
obj = Orchestrator.__new__(Orchestrator)
obj.stage_pools = [SimpleNamespace(final_output=False), _CodecPool(output_kind, registered, replica_id)]
obj.request_states = {"r": OrchestratorRequestState(request_id="r", final_stage_id=final_stage_id)}
obj.output_async_queue = asyncio.Queue()
obj._cfg_tracker = CfgCompanionTracker()
obj._pd_kv_params = {}
obj._running_counter = None
obj._abort_request_ids = AsyncMock(return_value=[])
obj._release_request_bindings = Mock()
obj._finish_raw_terminal_requests = AsyncMock()
async def process(stage, replica, raw, terminals):
await obj._route_upstream_first_audio(stage, replica, raw)
return await obj.stage_pools[stage].process_llm_raw_outputs(replica, raw)
async def route(stage, replica, outputs):
for output in outputs:
await obj.output_async_queue.put(
OutputMessage(
request_id=output.request_id,
stage_id=stage,
replica_id=replica,
engine_outputs=output,
metrics=None,
finished=output.finished,
)
)
obj._process_llm_stage_outputs = process
obj._handle_processed_outputs = route
return obj
def _raw(*, first=False, required=False, terminal=False, samples=(1, 1, 1, 1)):
return OmniEngineCoreOutputs(
outputs=[
OmniEngineCoreOutput(
request_id="r",
new_token_ids=[],
finish_reason=FinishReason.STOP if terminal else None,
multimodal_output={
"model_outputs": torch.tensor(samples, dtype=torch.float32),
"sr": torch.tensor(24000),
"_omni_first_audio": torch.tensor(first),
"_omni_first_audio_required": torch.tensor(required),
},
)
]
)
async def _codec_output(obj, raw):
replica = obj.stage_pools[1].replica_id
processed = await obj._process_llm_stage_outputs(1, replica, raw, set())
await obj._handle_processed_outputs(1, replica, processed)
def _messages(obj):
messages = []
while not obj.output_async_queue.empty():
messages.append(obj.output_async_queue.get_nowait())
return messages
def _audio(output):
audio = output.outputs[0].multimodal_output["audio"]
return torch.cat(audio) if isinstance(audio, list) else audio
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", list(RequestOutputKind))
@pytest.mark.parametrize("codec_overtook", [False, True])
async def test_first_audio_obeys_codec_output_kind(kind, codec_overtook):
obj = _orchestrator(kind)
suffix = _raw(required=True, samples=(20, 21))
terminal = _raw(terminal=True, samples=(30, 31))
if codec_overtook:
await _codec_output(obj, suffix)
await _codec_output(obj, terminal)
assert obj.output_async_queue.empty()
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
if kind == RequestOutputKind.FINAL_ONLY and not codec_overtook:
assert obj.output_async_queue.empty()
if not codec_overtook:
await _codec_output(obj, suffix)
await _codec_output(obj, terminal)
messages = _messages(obj)
# The source replica is zero; output ownership belongs to codec replica two.
assert all((message.stage_id, message.replica_id) == (1, 2) for message in messages)
actual = [_audio(message.engine_outputs).tolist() for message in messages]
expected = {
RequestOutputKind.DELTA: [[10, 11], [20, 21], [30, 31]],
RequestOutputKind.CUMULATIVE: [[10, 11], [10, 11, 20, 21], [10, 11, 20, 21, 30, 31]],
RequestOutputKind.FINAL_ONLY: [[10, 11, 20, 21, 30, 31]],
}
assert actual == expected[kind]
assert messages[-1].finished
assert obj.request_states["r"].pending_first_audio_outputs == []
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
assert obj.output_async_queue.empty()
@pytest.mark.asyncio
async def test_first_audio_waits_for_codec_registration_and_ignores_duplicates():
obj = _orchestrator(registered=False)
req_state = obj.request_states["r"]
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(90, 91)))
assert obj.output_async_queue.empty()
assert not req_state.upstream_first_audio
assert req_state.pending_upstream_first_audio is not None
obj.stage_pools[1].register()
await obj._flush_upstream_first_audio(req_state)
messages = _messages(obj)
assert len(messages) == 1
assert _audio(messages[0].engine_outputs).tolist() == [10, 11]
assert req_state.upstream_first_audio
assert req_state.pending_upstream_first_audio is None
@pytest.mark.asyncio
@pytest.mark.parametrize("processing_failed", [False, True])
async def test_prewarm_flushes_first_audio_after_successful_submission(mocker, processing_failed):
obj = _orchestrator(registered=False, final_stage_id=2 if processing_failed else 1)
req_state = obj.request_states["r"]
req_state.sampling_params_list = [None] * (req_state.final_stage_id + 1)
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
pool = obj.stage_pools[1]
pool.stage_type = "llm"
pool.stage_vllm_config = SimpleNamespace(model_config=SimpleNamespace(hf_config=None))
obj.stage_pools[0].get_bound_replica_id = lambda request_id: 0
obj._stage_receives_async_chunks = lambda stage: True
obj._build_payload_sender_info = Mock(return_value={})
obj._on_stage_submitted = Mock()
obj._emit_tx_edge = Mock()
mocker.patch(
"vllm_omni.engine.orchestrator.build_engine_core_request_from_tokens",
return_value=SimpleNamespace(request_id="r"),
)
async def submit(*args, **kwargs):
pool.register()
pool.submit_initial = submit
if processing_failed:
pool.process_llm_raw_outputs = AsyncMock(side_effect=RuntimeError("processing failed"))
later_pool = SimpleNamespace(submit_initial=AsyncMock())
obj.stage_pools.append(later_pool)
submitted = await obj._prewarm_async_chunk_stages("r", SimpleNamespace(prompt_token_ids=[0]), req_state)
assert submitted is not processing_failed
messages = _messages(obj)
assert len(messages) == 1
if processing_failed:
assert isinstance(messages[0], ErrorMessage)
later_pool.submit_initial.assert_not_called()
assert "r" not in obj.request_states
else:
assert _audio(messages[0].engine_outputs).tolist() == [10, 11]
@pytest.mark.asyncio
async def test_pending_first_audio_is_discarded_on_cancellation():
obj = _orchestrator(registered=False)
req_state = obj.request_states["r"]
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
await obj._cleanup_request_ids(["r"], abort=True)
obj.stage_pools[1].register()
await obj._flush_upstream_first_audio(req_state)
assert obj.output_async_queue.empty()
@pytest.mark.asyncio
async def test_cancellation_while_routing_first_audio_does_not_release_codec_outputs():
obj = _orchestrator()
await obj._route_upstream_first_audio(1, 2, _raw(required=True, terminal=True))
async def cancel(stage, replica, outputs):
await obj._cleanup_request_ids(["r"], abort=True)
obj._handle_processed_outputs = cancel
obj._process_llm_stage_outputs = AsyncMock()
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
obj._process_llm_stage_outputs.assert_not_called()
assert obj.output_async_queue.empty()
@pytest.mark.asyncio
async def test_request_ending_at_source_does_not_emit_first_audio():
obj = _orchestrator(final_stage_id=0)
raw = _raw(first=True)
await obj._route_upstream_first_audio(0, 0, raw)
assert not raw.outputs and obj.output_async_queue.empty()
assert obj.request_states["r"].pending_upstream_first_audio is None
@pytest.mark.asyncio
async def test_later_aligner_receives_full_codec_waveform():
from vllm_omni.model_executor.stage_input_processors.forced_aligner import code2wav2aligner
obj = _orchestrator(RequestOutputKind.CUMULATIVE, final_stage_id=2)
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
await _codec_output(obj, _raw(required=True, terminal=True, samples=(20, 21)))
messages = _messages(obj)
assert all(message.stage_id == 1 for message in messages)
inputs = code2wav2aligner(
[messages[-1].engine_outputs], prompt={"additional_information": {"text": ["Hello world"]}}
)
assert inputs[0]["multi_modal_data"]["audio"][0].tolist() == [10, 11, 20, 21]
@pytest.mark.asyncio
async def test_first_audio_processing_failure_fails_only_its_request():
obj = _orchestrator()
obj.request_states["healthy"] = OrchestratorRequestState(request_id="healthy", final_stage_id=1)
obj.stage_pools[1].process_llm_raw_outputs = AsyncMock(side_effect=RuntimeError("processing failed"))
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
error = obj.output_async_queue.get_nowait()
assert isinstance(error, ErrorMessage)
assert error.request_id == "r" and error.stage_id == 1
assert "processing failed" in error.error
assert set(obj.request_states) == {"healthy"}
@pytest.mark.asyncio
async def test_unmarked_upstream_audio_and_regular_codec_outputs_are_untouched():
obj = _orchestrator()
for stage in (0, 1):
raw = _raw()
await obj._route_upstream_first_audio(stage, 0, raw)
assert len(raw.outputs) == 1
assert torch.equal(raw.outputs[0].multimodal_output["model_outputs"], torch.ones(4))
assert obj.output_async_queue.empty()
@pytest.mark.asyncio
async def test_cancelled_request_does_not_receive_late_first_audio():
obj = _orchestrator()
obj.request_states.clear()
raw = _raw(first=True)
await obj._route_upstream_first_audio(0, 0, raw)
assert not raw.outputs and obj.output_async_queue.empty()
@pytest.mark.asyncio
async def test_first_audio_supports_both_speech_and_chat_consumers():
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
obj = _orchestrator()
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
result = obj.output_async_queue.get_nowait().engine_outputs
assert not result.finished
assert _audio(result).tolist() == [10, 11]
payload, key = OmniOpenAIServingSpeech._extract_audio_output(result)
assert payload is not None and key is not None
audio = payload[key]
audio = torch.cat(audio) if isinstance(audio, list) else audio
assert audio.tolist() == [10, 11]
@pytest.mark.asyncio
@pytest.mark.parametrize("audio", [None, [], torch.empty(0)])
@pytest.mark.parametrize("codec_overtook", [False, True])
async def test_invalid_first_audio_reports_error_and_releases_request(audio, codec_overtook):
obj = _orchestrator()
obj.request_states["healthy"] = OrchestratorRequestState(request_id="healthy", final_stage_id=1)
if codec_overtook:
await obj._route_upstream_first_audio(1, 2, _raw(required=True, terminal=True))
assert obj.request_states["r"].pending_first_audio_outputs
first = _raw(first=True)
first.outputs[0].multimodal_output["model_outputs"] = audio
await obj._route_upstream_first_audio(0, 0, first)
error = obj.output_async_queue.get_nowait()
assert isinstance(error, ErrorMessage)
assert error.request_id == "r" and "first-audio" in error.error
assert not first.outputs and set(obj.request_states) == {"healthy"}
obj._abort_request_ids.assert_awaited_once_with(["r"])
obj._release_request_bindings.assert_called_once_with(["r"])
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
assert obj.output_async_queue.empty()
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Cache isolation and graph replay for the opt-in TTS predictor."""
import pytest
import torch
from vllm_omni.model_executor.models.common.qwen3_code_predictor import CodePredictorBaseModel
from vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts import Qwen3TTSTalkerCodePredictorConfig
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
@pytest.mark.skipif(not torch.cuda.is_available() or torch.version.hip is not None, reason="requires CUDA")
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_cached_frames_match_causal_prefill_after_graph_replay(dtype):
from vllm_omni.model_executor.models.qwen3_tts.cached_code_predictor import FrameLocalKVCache
config = Qwen3TTSTalkerCodePredictorConfig(
vocab_size=64,
hidden_size=64,
intermediate_size=128,
num_hidden_layers=2,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=16,
num_code_groups=4,
)
torch.manual_seed(171)
model = CodePredictorBaseModel(config).to(device="cuda", dtype=dtype).eval()
inputs = torch.randn(3, 5, 64, device="cuda", dtype=dtype)
positions = torch.arange(5, device="cuda").expand(3, -1)
cache = FrameLocalKVCache(model, max_batch=3)
with torch.inference_mode():
for batch in (1, 3):
for _ in range(2):
for step in (1, 2, 3):
cache(inputs, batch, step)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = torch.cat([cache(inputs, 3, step) for step in (1, 2, 3)], dim=1)
for _ in range(3):
inputs.normal_()
# No previous frame (including a smaller batch) may leak through.
cache(inputs, 1, 1)
cache.buffer.fill_(torch.nan)
graph.replay()
expected = model(inputs, positions)[:, :4]
tolerance = 1e-5 if dtype == torch.float32 else 0.04
torch.testing.assert_close(output, expected, atol=tolerance, rtol=tolerance)
@@ -681,6 +681,7 @@ class TestCodePredictorGraphReplay:
sampling_mode="per_call",
return_proj_buf=False,
)
predictor._frame_cache = None
predictor._prefix_graphs_enabled = False
predictor._prefix_reprefill_enabled = False
predictor._bucket_pos_ids = {1: torch.arange(4).unsqueeze(0)}
@@ -1398,3 +1399,36 @@ class TestMTPExecutionBuckets:
)
wrapper2._bucket_sizes = [1, 2, 4] # stand-in for warmed-up state
wrapper2.configure_mtp_execution_buckets([3])
@pytest.mark.parametrize("enabled", [False, True])
def test_tts_cache_falls_back_on_cpu(mocker, loaded_target_classes, enabled):
cp_config, talker_config = _make_tiny_config(loaded_target_classes)
cfg = _make_vllm_config(mocker)
cfg.model_config.stage_connector_config = {"extra": {"code_predictor_kv_cache": enabled}}
predictor = loaded_target_classes[2](vllm_config=cfg, config=cp_config, talker_config=talker_config)
common = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"]
setup = mocker.patch.object(common.CodePredictorWrapper, "_setup_compile")
predictor._setup_compile()
setup.assert_called_once_with()
assert predictor._frame_cache is None
def test_shared_predictor_ignores_tts_fast_path_options(mocker, loaded_target_classes):
cp_config, _ = _make_tiny_config(loaded_target_classes)
cfg = _make_vllm_config(mocker)
cfg.model_config.stage_connector_config = {
"extra": {"code_predictor_kv_cache": True, "code_predictor_fused_sampling": True}
}
common = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"]
predictor = common.CodePredictorWrapper(
vllm_config=cfg,
cp_config=cp_config,
wrapper_config=common.CodePredictorWrapperConfig(),
)
assert not hasattr(predictor, "_frame_cache")
logits = torch.randn(3, cp_config.vocab_size)
uniforms = torch.rand_like(logits)
expected = (logits.float() - torch.log(-torch.log(uniforms))).argmax(-1, keepdim=True)
actual = predictor._sample_per_call(logits, 1.0, 0, None, uniforms)
torch.testing.assert_close(actual, expected)
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Supplied-uniform samples must match the existing ATen path exactly."""
import pytest
import torch
pytestmark = [
pytest.mark.core_model,
pytest.mark.cuda,
pytest.mark.skipif(not torch.cuda.is_available() or torch.version.hip is not None, reason="requires CUDA"),
]
@pytest.mark.parametrize("batch", [1, 64])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
@pytest.mark.parametrize("top_k", [-1, 1, 50, 2048])
def test_topk_gumbel_exact_samples_and_graph(batch, dtype, top_k):
from vllm_omni.model_executor.models.qwen3_tts.code_predictor_sampling import sample_code_topk_gumbel
with torch.inference_mode():
torch.manual_seed(42)
logits = torch.randn(batch, 2048, device="cuda", dtype=dtype) * 5
uniforms = torch.rand(batch, 15, 2048, device="cuda")[:, 7, :]
uniforms.clamp_(1e-6, 1 - 1e-6)
for case in ("random", "ties", "all_masked", "part_masked", "uniform_zero", "uniform_one", "nan"):
if case == "ties":
logits.zero_()
uniforms.fill_(0.5)
elif case == "all_masked":
logits.fill_(-torch.inf)
elif case == "part_masked":
logits[:, 4:9] = 1
elif case == "uniform_zero":
logits.normal_()
uniforms.zero_()
elif case == "uniform_one":
uniforms.fill_(1)
elif case == "nan":
uniforms.fill_(0.5)
logits[:, 7] = torch.nan
scaled = logits * (1 / 0.9)
if top_k > 0:
threshold = scaled.topk(top_k, dim=-1).values[:, -1:]
scaled = scaled.masked_fill(scaled < threshold, -torch.inf)
expected = (scaled.float() - torch.log(-torch.log(uniforms))).argmax(-1, keepdim=True)
actual = sample_code_topk_gumbel(logits, uniforms, top_k, 1 / 0.9)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = sample_code_topk_gumbel(logits, uniforms, top_k, 1 / 0.9)
graph.replay()
torch.testing.assert_close(captured, expected, atol=0, rtol=0)
@pytest.mark.parametrize("top_k", [0, 1, 50, 97])
def test_topk_gumbel_accepts_strided_supplied_uniforms(top_k):
from vllm_omni.model_executor.models.qwen3_tts.code_predictor_sampling import sample_code_topk_gumbel
torch.manual_seed(81)
logits = torch.randn(3, 194, device="cuda", dtype=torch.bfloat16)[:, ::2]
uniforms = torch.rand(3, 15, 194, device="cuda")[:, 3, ::2].clamp_(1e-6, 1 - 1e-6)
scaled = logits * (1 / 0.9)
if top_k > 0:
scaled = scaled.masked_fill(scaled < scaled.topk(top_k).values[:, -1:], -torch.inf)
expected = (scaled.float() - torch.log(-torch.log(uniforms))).argmax(-1, keepdim=True)
actual = sample_code_topk_gumbel(logits, uniforms, top_k, 1 / 0.9)
torch.testing.assert_close(actual, expected)
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Prefix graphs must follow per-request first-audio delivery, not a capture hint."""
from types import SimpleNamespace
import pytest
import torch
from tests.helpers.mark import hardware_test
from tests.model_executor.models.qwen3_tts.test_qwen3_tts_incremental_decode import _decoder_stub
from vllm_omni.model_executor.models.qwen3_tts.segmented_graph_wrapper import CUDAGraphDecoderWrapper
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
Qwen3TTSTokenizerV2Decoder,
)
pytestmark = [pytest.mark.core_model]
DEVICE = torch.device("cuda:0")
@pytest.mark.cpu
@pytest.mark.parametrize("skip_flags", [(False, False), (True, True), (False, True), (True, False)])
@pytest.mark.parametrize("initial_frames", [1, 3])
@pytest.mark.parametrize("state_only_available", [False, True])
def test_xvec_first_chunk_replays_prefix_graph(monkeypatch, skip_flags, initial_frames, state_only_available):
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
wrapper = CUDAGraphDecoderWrapper.__new__(CUDAGraphDecoderWrapper)
wrapper.prefix_length = 72
wrapper.initial_chunk_frames = initial_frames
wrapper.codec_chunk_frames = 25
wrapper.capture_batch_sizes = [1, 2]
wrapper._icl_previous_frames_by_target = {26: 1}
wrapper._xvec_previous_frames_by_target = {initial_frames + 25: initial_frames}
replayed = []
wrapper.xvec_prefix_states = {
2: {
"graph": SimpleNamespace(replay=lambda: replayed.append("full")),
"input": {"codes": torch.zeros(2, 2, initial_frames)},
"output": torch.arange(4 * initial_frames, dtype=torch.float32).view(2, 1, 2 * initial_frames),
"cache": {
"ref_hidden": torch.zeros(2, 2, 0),
"ref_conv": torch.zeros(2, 0, 3),
"prefix_hidden": torch.zeros(2, 0, 3),
"ref_upsample": torch.zeros(2, 3, 0),
"ref_wav": torch.zeros(2, 1, 0),
"suffix_quantized": torch.ones(2, 2, initial_frames),
"suffix_conv": torch.ones(2, initial_frames, 3),
"past_key_values": SimpleNamespace(layers=[]),
},
}
}
wrapper.xvec_prefix_state_only_states = {}
if state_only_available and initial_frames == 1:
wrapper.xvec_prefix_state_only_states[2] = {
**wrapper.xvec_prefix_states[2],
"graph": SimpleNamespace(replay=lambda: replayed.append("state")),
"state_only": True,
}
wrapper._record_graph_hit = lambda *_args: None
wrapper._record_graph_fallback = lambda *_args: None
wrapper._ensure_suffix_buffers = lambda cache: None
wrapper.decoder = _decoder_stub(
capture_first_audio_state_only=True,
total_upsample=2,
_is_suffix_cache_rolling=lambda previous, cached: False,
_decode_xvec_first_chunk=lambda *_args: pytest.fail("unexpected eager fallback"),
_slice_dynamic_cache=Qwen3TTSTokenizerV2Decoder._slice_dynamic_cache,
)
caches = [{"prefix_frames": 0, "skip_first_audio": skip} for skip in skip_flags]
outputs = wrapper._batched_request_decode(
[torch.full((1, 2, initial_frames), 3), torch.full((1, 2, initial_frames), 4)],
caches,
)
assert replayed == ["state" if all(skip_flags) and state_only_available and initial_frames == 1 else "full"]
torch.testing.assert_close(wrapper.xvec_prefix_states[2]["input"]["codes"][0], torch.full((2, initial_frames), 3.0))
torch.testing.assert_close(wrapper.xvec_prefix_states[2]["input"]["codes"][1], torch.full((2, initial_frames), 4.0))
assert [cache["decoder_prefix_frames"] for cache in caches] == [0, 0]
assert [cache["suffix_frames"] for cache in caches] == [initial_frames, initial_frames]
assert len(outputs) == 2
for row, skip in enumerate(skip_flags):
expected = wrapper.xvec_prefix_states[2]["output"][row : row + 1, :, 2 if skip else 0 :]
torch.testing.assert_close(outputs[row], expected)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("initial_frames", [1, 3])
@pytest.mark.parametrize("time_major", [False, True])
@torch.inference_mode()
def test_xvec_graphs_follow_request_first_audio_flags(initial_frames, time_major):
from tests.model_executor.models.qwen3_tts.test_time_major_decoder import _make_decoder
from vllm_omni.model_executor.models.qwen3_tts.segmented_graph_wrapper import (
CUDAGraphDecoderWrapper as SegmentedWrapper,
)
decoder = _make_decoder().to(DEVICE)
decoder.precompute_snake_caches()
if time_major:
decoder.enable_time_major_conv()
decoder.capture_first_audio_state_only = True
wrapper = SegmentedWrapper(
decoder,
capture_modes=("xvec",),
capture_batch_sizes=[1, 2],
num_quantizers=2,
initial_chunk_frames=initial_frames,
)
# This regression covers prefix graphs. Avoid compiling unrelated suffix
# shapes, especially the time-major kernels for long continuation chunks.
wrapper._xvec_previous_frames_by_target = {}
wrapper.warmup(DEVICE)
assert set(wrapper.xvec_prefix_states) == {1, 2}
assert set(wrapper.xvec_prefix_state_only_states) == ({1, 2} if initial_frames == 1 else set())
for skip_flags in ((False, False), (True, True), (False, True), (True, False)):
codes = [torch.randint(0, 32, (1, 2, initial_frames), device=DEVICE) for _ in skip_flags]
caches = [{"prefix_frames": 0, "skip_first_audio": skip} for skip in skip_flags]
expected_caches = [{"prefix_frames": 0, "skip_first_audio": skip} for skip in skip_flags]
expected = [decoder._decode_stream_first_chunk(code, cache) for code, cache in zip(codes, expected_caches)]
actual = wrapper._decode_xvec_prefix_batch(codes, caches)
assert actual is not None # Eligible batches must not silently become per-request eager calls.
for got, want, cache, expected_cache in zip(actual, expected, caches, expected_caches):
torch.testing.assert_close(got, want, atol=1e-4, rtol=1e-4)
for key in ("ref_hidden", "ref_conv", "prefix_hidden", "suffix_quantized", "suffix_conv"):
torch.testing.assert_close(cache[key], expected_cache[key], atol=1e-5, rtol=1e-5)
@@ -752,3 +752,37 @@ def test_load_weights_uses_model_dtype_before_precomputing_caches(dtype):
assert _load_weights_noop(model) == {"decoder.fake_weight"}
assert model.decoder.weight.dtype is dtype
assert cache_dtypes == [dtype]
@pytest.mark.parametrize("capture_fails", [False, True])
def test_decode_autotune_restores_process_flags(mocker, monkeypatch, capture_fails):
monkeypatch.setattr(torch.backends.cudnn, "benchmark", False)
monkeypatch.setattr(torch.backends.cudnn, "benchmark_limit", 5)
original = (torch.backends.cudnn.enabled, torch.backends.cudnn.deterministic, torch.backends.cudnn.allow_tf32)
model = _make_model(
async_chunk=True,
device=torch.device("cuda"),
stage_connector_config={"extra": {"decode_cudnn_benchmark": True}},
)
observed = []
def capture(**kwargs):
observed.append(
(
torch.backends.cudnn.benchmark,
torch.backends.cudnn.benchmark_limit,
torch.backends.cudnn.enabled,
torch.backends.cudnn.deterministic,
torch.backends.cudnn.allow_tf32,
)
)
if capture_fails:
raise RuntimeError("capture failed")
capture_mock = mocker.patch.object(model, "_maybe_enable_decoder_cudagraph", side_effect=capture)
_load_weights_noop(model)
capture_mock.assert_called_once()
assert observed == [(True, 10, *original)]
assert torch.backends.cudnn.benchmark is False
assert torch.backends.cudnn.benchmark_limit == 5
@@ -0,0 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Causal/GQA correctness and graph replay for the frame-local KV kernel."""
import pytest
import torch
pytestmark = [
pytest.mark.core_model,
pytest.mark.cuda,
pytest.mark.skipif(not torch.cuda.is_available() or torch.version.hip is not None, reason="requires CUDA"),
]
@pytest.mark.parametrize("batch", [1, 3, 64])
@pytest.mark.parametrize("heads,kv_heads,dim", [(16, 8, 128), (8, 8, 64)])
@pytest.mark.parametrize("positions", [[0, 1], [2], [8], [15]])
def test_short_attention_causal_gqa_and_graph(batch, heads, kv_heads, dim, positions):
from vllm_omni.model_executor.models.qwen3_tts.short_kv_attention import short_kv_attention
with torch.inference_mode():
torch.manual_seed(42)
pos = torch.tensor(positions, device="cuda")
q = torch.randn(batch, len(positions), heads, dim, device="cuda", dtype=torch.bfloat16).transpose(1, 2)
cache = torch.randn(2, batch + 3, kv_heads, 17, dim, device="cuda", dtype=torch.bfloat16)
k, v = cache[0, :batch], cache[1, :batch]
mask = torch.arange(17, device="cuda")[None, :] <= pos[:, None]
kr = k.float().repeat_interleave(heads // kv_heads, dim=1)
vr = v.float().repeat_interleave(heads // kv_heads, dim=1)
scores = (q.float() @ kr.transpose(-1, -2)) * dim**-0.5
expected = scores.masked_fill(~mask, -torch.inf).softmax(-1) @ vr
actual = short_kv_attention(q, k, v, pos, dim**-0.5)
torch.testing.assert_close(actual.float(), expected, atol=0.016, rtol=0.008)
# Unwritten cache slots must not affect a new frame, even if poisoned.
k[:, :, max(positions) + 1 :] = torch.nan
v[:, :, max(positions) + 1 :] = torch.nan
poisoned = short_kv_attention(q, k, v, pos, dim**-0.5)
torch.testing.assert_close(poisoned, actual, atol=0, rtol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = short_kv_attention(q, k, v, pos, dim**-0.5)
graph.replay()
torch.testing.assert_close(captured, actual, atol=0, rtol=0)
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Code2Wav drops only frames explicitly marked as delivered by the Talker."""
import pytest
import torch
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
Qwen3TTSTokenizerV2Decoder,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
UPSAMPLE = 4
def _decoder(skip: bool):
decoder = Qwen3TTSTokenizerV2Decoder.__new__(Qwen3TTSTokenizerV2Decoder)
decoder.__dict__["capture_first_audio_state_only"] = skip
decoder.__dict__["total_upsample"] = UPSAMPLE
calls = []
def full(codes, caches):
calls.append("full")
frames = int(codes.shape[-1])
return torch.arange(frames * UPSAMPLE, dtype=torch.float32).view(1, 1, -1)
def state_only(codes, caches):
calls.append("state")
return torch.zeros(1)
decoder.__dict__["_decode_xvec_first_chunk"] = full
decoder.__dict__["_decode_xvec_first_chunk_state_only"] = state_only
return decoder, calls
def test_one_frame_first_chunk_only_builds_state():
decoder, calls = _decoder(skip=True)
out = decoder._decode_stream_first_chunk(torch.zeros(1, 16, 1, dtype=torch.long), {"skip_first_audio": True})
assert out.shape[-1] == 0 and calls == ["state"]
def test_longer_first_chunk_keeps_all_but_the_delivered_frame():
decoder, calls = _decoder(skip=True)
out = decoder._decode_stream_first_chunk(
torch.zeros(1, 16, 3, dtype=torch.long),
{"skip_first_audio": True},
)
assert calls == ["full"]
assert torch.equal(out, torch.arange(UPSAMPLE, 3 * UPSAMPLE, dtype=torch.float32).view(1, 1, -1))
def test_without_skip_the_whole_first_chunk_is_decoded():
decoder, calls = _decoder(skip=False)
out = decoder._decode_stream_first_chunk(
torch.zeros(1, 16, 3, dtype=torch.long),
{"skip_first_audio": False},
)
assert calls == ["full"] and out.shape[-1] == 3 * UPSAMPLE
def test_request_without_first_audio_uses_regular_decoder_even_when_graph_skip_is_enabled():
decoder, calls = _decoder(skip=True)
out = decoder._decode_stream_first_chunk(torch.zeros(1, 16, 1, dtype=torch.long), {})
assert calls == ["full"] and out.shape[-1] == UPSAMPLE
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
import torch.nn.functional as F
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz import time_major_conv as tmc
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _snake_nct(x, exp_alpha, inv_beta):
return x + inv_beta[None, :, None] * torch.sin(x * exp_alpha[None, :, None]).square()
@pytest.mark.parametrize(("kernel_size", "dilation"), [(1, 1), (7, 1), (7, 3), (3, 9)])
@pytest.mark.parametrize("with_bias_and_snake", [False, True])
def test_im2col_gemm_matches_causal_conv1d(kernel_size, dilation, with_bias_and_snake):
torch.manual_seed(0)
bsz, t, c_in, c_out = 2, 11, 6, 5
conv = torch.nn.Conv1d(c_in, c_out, kernel_size, dilation=dilation)
x = torch.randn(bsz, c_in, t)
in_bias = torch.randn(c_in) if with_bias_and_snake else None
snake = (torch.rand(c_in) + 0.5, torch.rand(c_in) + 0.5) if with_bias_and_snake else None
ref_in = x if in_bias is None else x + in_bias[None, :, None]
if snake is not None:
ref_in = _snake_nct(ref_in, *snake)
expected = conv(F.pad(ref_in, ((kernel_size - 1) * dilation, 0)))
cols = tmc.snake_im2col(x.transpose(1, 2).contiguous(), kernel_size, dilation, in_bias, snake)
actual = torch.addmm(conv.bias, cols, tmc.conv_gemm_weight(conv.weight)).view(bsz, t, c_out)
torch.testing.assert_close(actual.transpose(1, 2), expected, rtol=1e-5, atol=1e-5)
@pytest.mark.parametrize(("stride", "kernel_size"), [(2, 2), (3, 6), (8, 16)])
def test_transposed_overlap_add_matches_trimmed_conv_transpose(stride, kernel_size):
torch.manual_seed(0)
bsz, t, c_in, c_out = 2, 5, 4, 3
conv = torch.nn.ConvTranspose1d(c_in, c_out, kernel_size, stride=stride)
x = torch.randn(bsz, c_in, t)
expected = conv(x)[..., : t * stride] # causal: drop the kernel - stride right tail
z = (x.transpose(1, 2).reshape(bsz * t, c_in) @ tmc.transposed_gemm_weight(conv.weight)).view(
bsz, t, kernel_size, c_out
)
actual = tmc.transposed_overlap_add(z, stride, conv.bias)
torch.testing.assert_close(actual.transpose(1, 2), expected, rtol=1e-5, atol=1e-5)
def test_transposed_overlap_add_rejects_other_kernels():
with pytest.raises(ValueError, match="kernel stride or 2"):
tmc.transposed_overlap_add(torch.zeros(1, 2, 5, 3), 2, torch.zeros(3))
@pytest.mark.parametrize(("kernel_size", "dilation"), [(1, 1), (7, 3)])
@pytest.mark.parametrize("with_residual", [False, True])
def test_causal_conv_matches_conv1d(kernel_size, dilation, with_residual):
torch.manual_seed(0)
bsz, t, c_in, c_out = 2, 13, 6, 5
conv = torch.nn.Conv1d(c_in, c_out, kernel_size, dilation=dilation)
x = torch.randn(bsz, c_in, t)
residual = torch.randn(bsz, t, c_out) if with_residual else None
expected = conv(F.pad(x, ((kernel_size - 1) * dilation, 0))).transpose(1, 2)
if residual is not None:
expected = expected + residual
actual = tmc.causal_conv(
x.transpose(1, 2).contiguous(), tmc.conv_gemm_weight(conv.weight), conv.bias, kernel_size, dilation, residual
)
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
def _gpu_tolerances(dtype):
# bf16 results may differ by one ulp (sin rounding, fp32 vs bf16 accumulation order).
return {"rtol": 1.6e-2, "atol": 1e-2} if dtype == torch.bfloat16 else {"rtol": 1e-4, "atol": 1e-4}
@pytest.mark.skipif(not torch.cuda.is_available() or not tmc.HAS_TRITON, reason="needs CUDA and Triton")
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_triton_kernels_match_reference(dtype):
torch.manual_seed(0)
tol = _gpu_tolerances(dtype)
x = torch.randn(3, 37, 96, device="cuda", dtype=dtype)
bias = torch.randn(96, device="cuda", dtype=dtype)
snake = (torch.rand(96, device="cuda", dtype=dtype) + 0.5, torch.rand(96, device="cuda", dtype=dtype) + 0.5)
for kernel_size, dilation in ((1, 1), (7, 9)):
torch.testing.assert_close(
tmc.snake_im2col(x, kernel_size, dilation, bias, snake),
tmc.snake_im2col_reference(x, kernel_size, dilation, bias, snake),
**tol,
)
z = torch.randn(3, 37, 10, 48, device="cuda", dtype=dtype)
torch.testing.assert_close(
tmc.transposed_overlap_add(z, 5, bias[:48]), tmc.transposed_overlap_add_reference(z, 5, bias[:48]), **tol
)
@pytest.mark.skipif(not torch.cuda.is_available() or not tmc.HAS_TRITON, reason="needs CUDA and Triton")
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize(("c_in", "c_out", "kernel_size", "dilation"), [(96, 96, 7, 9), (192, 96, 1, 1), (96, 1, 7, 1)])
def test_triton_causal_conv_matches_reference(dtype, c_in, c_out, kernel_size, dilation):
torch.manual_seed(0)
x = torch.randn(3, 101, c_in, device="cuda", dtype=dtype)
weight = torch.randn(kernel_size * c_in, c_out, device="cuda", dtype=dtype) / (kernel_size * c_in) ** 0.5
bias = torch.randn(c_out, device="cuda", dtype=dtype)
residual = torch.randn(3, 101, c_out, device="cuda", dtype=dtype)
expected = tmc.causal_conv_reference(
x.float(), weight.float(), bias.float(), kernel_size, dilation, residual.float()
)
actual = tmc.causal_conv(x, weight, bias, kernel_size, dilation, residual)
torch.testing.assert_close(actual.float(), expected, **_gpu_tolerances(dtype))
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import (
Qwen3TTSTokenizerV2DecoderConfig,
)
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
Qwen3TTSTokenizerV2Decoder,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_decoder() -> Qwen3TTSTokenizerV2Decoder:
config = Qwen3TTSTokenizerV2DecoderConfig(
codebook_size=32,
hidden_size=16,
latent_dim=16,
codebook_dim=16,
num_attention_heads=2,
num_key_value_heads=2,
intermediate_size=32,
num_hidden_layers=1,
num_quantizers=2,
decoder_dim=32,
upsample_rates=(8, 5, 4, 3),
upsampling_ratios=(2, 2),
sliding_window=72,
)
torch.manual_seed(0)
decoder = Qwen3TTSTokenizerV2Decoder(config).eval()
with torch.no_grad():
for name, param in decoder.named_parameters():
# Non-trivial SnakeBeta and ConvNeXt scales so every term matters.
if name.endswith(("alpha", "beta")):
param.uniform_(-0.5, 0.5)
elif name.endswith("gamma"):
param.uniform_(0.5, 1.0)
decoder.precompute_snake_caches()
return decoder
@torch.inference_mode()
def test_time_major_conv_decode_matches_nct_stack() -> None:
decoder = _make_decoder()
hidden = torch.randn(2, 5, decoder.config.latent_dim)
expected = decoder._conv_decode(hidden)
decoder.enable_time_major_conv()
actual = decoder._conv_decode(hidden)
assert actual.shape == expected.shape == (2, 1, 5 * int(decoder.total_upsample))
scale = expected.abs().max()
torch.testing.assert_close(actual / scale, expected / scale, rtol=0, atol=1e-5)
# The stack holds weight copies outside the module tree.
assert not any("time_major" in key for key in decoder.state_dict())
@torch.inference_mode()
def test_time_major_conv_decode_takes_sliced_hidden() -> None:
decoder = _make_decoder()
hidden = torch.randn(1, 9, decoder.config.latent_dim)
expected = decoder._conv_decode(hidden[:, -4:, :])
decoder.enable_time_major_conv()
actual = decoder._conv_decode(hidden[:, -4:, :]) # non-contiguous, like the suffix decode's slice
scale = expected.abs().max()
torch.testing.assert_close(actual / scale, expected / scale, rtol=0, atol=1e-5)
@torch.inference_mode()
def test_first_frame_single_key_attention_matches_codec_without_graphs():
from vllm_omni.model_executor.models.qwen3_tts.first_frame_decoder import Qwen3TTSFirstFrameDecoder
decoder = _make_decoder()
codes = torch.randint(0, 32, (3, 2, 1))
expected = decoder._decode_xvec_first_chunk(codes, {})[:, 0, :]
first = Qwen3TTSFirstFrameDecoder.__new__(Qwen3TTSFirstFrameDecoder)
torch.nn.Module.__init__(first)
first.decoder = decoder
first._graphs = {}
actual = first.decode(codes[:, :, 0])
torch.testing.assert_close(actual, expected, rtol=1e-4, atol=1e-5)
+237
View File
@@ -0,0 +1,237 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import queue
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.engine import OmniEngineCoreOutputs
from vllm_omni.worker_v2.first_audio_sender import engine_output_queue_sink
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _scheduler(**client_by_request):
return SimpleNamespace(
requests={request_id: SimpleNamespace(client_index=client) for request_id, client in client_by_request.items()}
)
def test_sink_groups_outputs_by_client_and_carries_audio():
output_queue: queue.Queue = queue.Queue()
sink = engine_output_queue_sink(output_queue, _scheduler(a=0, b=1, c=0))
sample_rate = torch.tensor(24000, dtype=torch.int32)
rows = [torch.full((4,), float(i)) for i in range(3)]
sink.prepare(["a", "b", "c"])(["a", "b", "c"], rows, sample_rate)
delivered = {}
while not output_queue.empty():
client_index, outputs = output_queue.get_nowait()
assert isinstance(outputs, OmniEngineCoreOutputs)
delivered[client_index] = outputs.outputs
assert sorted(delivered) == [0, 1]
assert [o.request_id for o in delivered[0]] == ["a", "c"]
assert [o.request_id for o in delivered[1]] == ["b"]
for output in delivered[0] + delivered[1]:
# First audio carries no tokens and does not finish the request.
assert output.new_token_ids == []
assert output.finish_reason is None
assert output.multimodal_output["sr"] is sample_rate
assert torch.equal(delivered[0][1].multimodal_output["model_outputs"], rows[2])
def test_sink_skips_requests_that_already_left_the_scheduler():
output_queue: queue.Queue = queue.Queue()
sink = engine_output_queue_sink(output_queue, _scheduler(kept=0))
sink.prepare(["gone", "kept"])(["gone", "kept"], [torch.zeros(2), torch.ones(2)], torch.tensor(24000))
sink.prepare(["gone"])(["gone"], [torch.zeros(2)], torch.tensor(24000))
client_index, outputs = output_queue.get_nowait()
assert client_index == 0
assert [o.request_id for o in outputs.outputs] == ["kept"]
assert output_queue.empty()
class _DoneEvent:
def record(self):
pass
def synchronize(self):
pass
def _run_sender(items):
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
output_queue: queue.Queue = queue.Queue()
sink = engine_output_queue_sink(output_queue, _scheduler(**{rid: 0 for item in items for rid in item[2]}))
sender = FirstAudioSender(sink)
for item in items:
sender._queue.put((*item, sink.prepare(item[2])))
sender.close()
delivered = []
while not output_queue.empty():
_, outputs = output_queue.get_nowait()
delivered.append(
([o.request_id for o in outputs.outputs], [o.multimodal_output["model_outputs"] for o in outputs.outputs])
)
return delivered
def test_sender_drops_rows_whose_frame_is_not_audio():
pcm = torch.arange(6, dtype=torch.float32).reshape(3, 2)
sample_rate = torch.tensor(24000, dtype=torch.int32)
valid = torch.tensor([True, False, True])
delivered = _run_sender(
[
(_DoneEvent(), pcm, ["a", "b", "c"], sample_rate, valid),
# Every row invalid: nothing reaches the sink.
(_DoneEvent(), pcm[:1], ["d"], sample_rate, torch.tensor([False])),
# No validity: every row is delivered (previous behaviour).
(_DoneEvent(), pcm[:1], ["e"], sample_rate, None),
]
)
assert [ids for ids, _rows in delivered] == [["a", "c"], ["e"]]
assert torch.equal(delivered[0][1][1], pcm[2])
def test_prepared_delivery_survives_scheduler_completion():
output_queue: queue.Queue = queue.Queue()
scheduler = _scheduler(r=3)
sink = engine_output_queue_sink(output_queue, scheduler)
deliver = sink.prepare(["r"])
scheduler.requests.clear()
deliver(["r"], [torch.ones(2)], torch.tensor(24000))
client, outputs = output_queue.get_nowait()
assert client == 3
assert outputs.outputs[0].multimodal_output["_omni_first_audio"]
def test_delivery_failure_emits_request_error():
from vllm.v1.engine import FinishReason
output_queue: queue.Queue = queue.Queue()
delivery = engine_output_queue_sink(output_queue, _scheduler(r=0)).prepare(["r"])
delivery.fail(["r"])
_, outputs = output_queue.get_nowait()
assert outputs.outputs[0].finish_reason == FinishReason.ERROR
@pytest.mark.parametrize("client_index", [0, 3])
def test_submit_reports_only_requests_with_a_prepared_route(monkeypatch, client_index):
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
# Exercise the routing contract without allocating CUDA or pinned memory.
original_empty = torch.empty
monkeypatch.setattr(torch, "empty", lambda *args, pin_memory=False, **kwargs: original_empty(*args, **kwargs))
monkeypatch.setattr(torch.cuda, "Event", _DoneEvent)
output_queue: queue.Queue = queue.Queue()
sender = FirstAudioSender(engine_output_queue_sink(output_queue, _scheduler(kept=client_index)))
pcm = torch.arange(6, dtype=torch.float32).reshape(3, 2)
try:
accepted = sender.submit(["gone", "kept", "also-gone"], pcm, torch.tensor(24000))
finally:
sender.close()
assert accepted == ["kept"]
client, outputs = output_queue.get_nowait()
assert client == client_index
assert [output.request_id for output in outputs.outputs] == ["kept"]
assert torch.equal(outputs.outputs[0].multimodal_output["model_outputs"], pcm[1])
assert output_queue.empty()
def test_submit_without_any_route_leaves_audio_to_the_codec(monkeypatch):
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
def unexpected_copy(*args, **kwargs):
pytest.fail("No delivery is possible; do not start a device copy")
monkeypatch.setattr(torch, "empty", unexpected_copy)
monkeypatch.setattr(torch.cuda, "Event", unexpected_copy)
output_queue: queue.Queue = queue.Queue()
sender = FirstAudioSender(engine_output_queue_sink(output_queue, _scheduler()))
try:
accepted = sender.submit(["gone"], torch.ones(1, 2), torch.tensor(24000))
finally:
sender.close()
assert accepted == []
assert output_queue.empty()
@pytest.mark.parametrize("sink", [lambda ids, rows, sr: None, SimpleNamespace(prepare=None)])
def test_sender_rejects_sink_without_route_preparation(sink):
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
with pytest.raises(TypeError, match="prepare"):
FirstAudioSender(sink)
def test_sender_failure_only_aborts_valid_undelivered_rows():
from vllm.v1.engine import FinishReason
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
class FailSecondClientOnce(queue.Queue):
def put_nowait(self, item):
client, outputs = item
if client == 1 and outputs.outputs[0].finish_reason is None:
raise RuntimeError("client enqueue failed")
super().put_nowait(item)
output_queue = FailSecondClientOnce()
sink = engine_output_queue_sink(output_queue, _scheduler(eos=1, sent=0, failed=1, pending=2, next=3))
sender = FirstAudioSender(sink)
ids = ["eos", "sent", "failed", "pending", "gone"]
sender._queue.put(
(
_DoneEvent(),
torch.ones(5, 2),
ids,
torch.tensor(24000),
torch.tensor([False, True, True, True, True]),
sink.prepare(ids),
)
)
sender._queue.put((_DoneEvent(), torch.ones(1, 2), ["next"], torch.tensor(24000), None, sink.prepare(["next"])))
sender.close()
actual: list[tuple[int, str, FinishReason | None]] = []
while not output_queue.empty():
client, outputs = output_queue.get_nowait()
actual.extend((client, o.request_id, o.finish_reason) for o in outputs.outputs)
assert actual == [
(0, "sent", None),
(1, "failed", FinishReason.ERROR),
(2, "pending", FinishReason.ERROR),
(3, "next", None),
]
def test_sender_copy_failure_reports_all_prepared_routes():
from vllm.v1.engine import FinishReason
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
class FailedEvent:
def synchronize(self):
raise RuntimeError("copy failed before validity became readable")
output_queue: queue.Queue = queue.Queue()
sink = engine_output_queue_sink(output_queue, _scheduler(r=3))
sender = FirstAudioSender(sink)
sender._queue.put(
(FailedEvent(), torch.ones(2, 2), ["r", "gone"], torch.tensor(24000), None, sink.prepare(["r", "gone"]))
)
sender.close()
client, outputs = output_queue.get_nowait()
assert client == 3
assert [(o.request_id, o.finish_reason) for o in outputs.outputs] == [("r", FinishReason.ERROR)]
assert output_queue.empty()
+201
View File
@@ -391,3 +391,204 @@ def test_mtp_requires_model_declared_output_key(output_key):
model = SimpleNamespace(mtp=lambda: None, mtp_output_key=output_key)
with pytest.raises(TypeError, match="must declare mtp_output_key"):
state._init_mtp_runner(model)
_EAGER_DIM = 3
_CODEBOOK = 2048
_EOS = 2150
class _EagerBatch(_DummyInputBatch):
def __init__(self, spans, indices=None):
indices = list(range(len(spans))) if indices is None else indices
super().__init__(indices)
starts = [0]
for n_tok in spans:
starts.append(starts[-1] + n_tok)
self.num_scheduled_tokens = list(spans)
self.query_start_loc_np = starts[:-1]
self.query_start_loc = torch.tensor(starts, dtype=torch.int32)
def _make_eager_state(max_num_reqs=4):
state = _make_state(max_num_reqs=max_num_reqs, has_preprocess=True)
_init_static(state, max_num_reqs, dim=_EAGER_DIM)
from vllm_omni.worker_v2.model_states.eager_mtp import EagerMTPState
state._eager_state = EagerMTPState(state)
state._first_audio_requests = set()
state._eager_mtp = True
state._eager_fastpath = False
state._eager_rows = None
state._eager_ready = {}
state._eager_settled = {}
state._eager_embeds = torch.zeros((max_num_reqs, _EAGER_DIM))
state._decode_preprocess = None
state._mtp_sample_uniforms = None
state.vllm_config.cache_config = SimpleNamespace(enable_prefix_caching=False)
model = state.model
model.first_frame_decoder = None
model.mtp_frame_valid = lambda layer0: (layer0 >= 0) & (layer0 < _CODEBOOK)
model.embed_input_ids = lambda ids: ids.float().reshape(-1, 1, 1).expand(-1, 1, _EAGER_DIM)
model.mtp_calls = []
def mtp(input_ids, input_embeds, last_hidden, text_step, **kwargs):
model.mtp_calls.append((input_ids.clone(), last_hidden.clone(), text_step.clone()))
codes = torch.stack([input_ids, input_ids + 1, input_ids + 2], dim=1)
# Frame embedding sum plus the text step, like the Qwen3 talkers.
return input_embeds + 100 + text_step, codes
model.mtp = mtp
return state
def _eager_outputs(num_tokens):
return {
"codes": {"audio": torch.zeros((num_tokens, 3), dtype=torch.long)},
"meta": {"codec_frame_valid": torch.zeros((num_tokens,), dtype=torch.int8)},
}
def test_eager_mtp_publishes_frames_in_the_step_that_sampled_cb0():
state = _make_eager_state()
_fill_buffers(state, "prefill", "decode", "ended")
# prefill: final 3-token chunk; decode: input CB0 42; ended: input CB0 was
# EOS (async scheduling ran the finished request once more).
batch = _EagerBatch([3, 1, 1])
input_ids = torch.tensor([0, 0, 0, 42, _EOS])
state._eager_rows = (batch, [(0, 0, "prefill", True), (1, 1, "decode", False), (2, 2, "ended", False)], input_ids)
text_hidden = torch.arange(5 * _EAGER_DIM, dtype=torch.float32).reshape(5, _EAGER_DIM)
sampled = torch.tensor([[7], [_EOS], [9]])
outputs = _eager_outputs(5)
state.run_eager_mtp(batch, text_hidden, sampled, outputs)
ids, hidden, text_step = state.model.mtp_calls[0]
assert ids.tolist() == [7, _EOS, 9]
# The frame's hidden is this step's last-token hidden of each span.
assert torch.equal(hidden, text_hidden[[2, 3, 4]])
# The next step adds its own text step.
assert not text_step.any()
codes = outputs["codes"]["audio"]
assert codes[2].tolist() == [7, 8, 9]
assert codes[3].tolist() == [_EOS, _EOS + 1, _EOS + 2]
assert codes[4].tolist() == [9, 10, 11]
assert not codes[:2].any()
# Valid only when both the sampled and (for decode) the input CB0 are codec ids.
assert outputs["meta"]["codec_frame_valid"].tolist() == [0, 0, 1, 0, 0]
assert state._eager_ready == {0: "prefill", 1: "decode", 2: "ended"}
assert torch.equal(state._eager_embeds[0], torch.full((_EAGER_DIM,), 107.0))
assert state._eager_rows is None
def test_eager_mtp_ignores_rows_recorded_for_another_batch():
state = _make_eager_state()
_fill_buffers(state, "r0")
state._eager_rows = (_EagerBatch([1]), [(0, 0, "r0", False)], torch.tensor([5]))
outputs = _eager_outputs(1)
state.run_eager_mtp(_EagerBatch([1]), torch.zeros(1, _EAGER_DIM), torch.tensor([[3]]), outputs)
assert state.model.mtp_calls == []
assert not outputs["codes"]["audio"].any()
def test_eager_mtp_requires_retained_frame_outputs():
state = _make_eager_state()
_fill_buffers(state, "r0")
batch = _EagerBatch([1])
state._eager_rows = (batch, [(0, 0, "r0", False)], torch.tensor([5]))
with pytest.raises(RuntimeError, match="requires retained codes.audio"):
state.run_eager_mtp(batch, torch.zeros(1, _EAGER_DIM), torch.tensor([[3]]), {"codes": {"audio": None}})
def test_eager_decode_input_is_previous_frame_plus_text_step():
state = _make_eager_state()
_fill_buffers(state, "r0", "r1")
state._eager_ready = {0: "r0", 1: "r1"}
state._eager_embeds[0] = 5.0
state._eager_embeds[1] = 7.0
embeds = torch.zeros((2, _EAGER_DIM))
batches = [
(0, 0, (torch.zeros(_EAGER_DIM), torch.full((_EAGER_DIM,), 2.0))),
(1, 1, (None, torch.ones(_EAGER_DIM))),
]
state._eager_state._apply_eager_frames(batches, embeds, _EagerBatch([1, 1]), None)
assert embeds.tolist() == [[7.0] * _EAGER_DIM, [8.0] * _EAGER_DIM]
# The deferred MTP must not run for an eager decode row.
assert state.model.mtp_calls == []
def test_eager_decode_without_a_frame_fails_loudly():
state = _make_eager_state()
_fill_buffers(state, "r0")
batches = [(0, 0, (torch.zeros(_EAGER_DIM), torch.zeros(_EAGER_DIM)))]
with pytest.raises(RuntimeError, match="frame missing"):
state._eager_state._apply_eager_frames(batches, torch.zeros((1, _EAGER_DIM)), _EagerBatch([1]), None)
def test_run_preprocess_records_rows_that_keep_a_sample():
state = _make_eager_state()
_fill_buffers(state, "chunk", "final", "decode")
state._eager_ready = {2: "decode"}
state._eager_embeds[2] = 4.0
text_step = torch.ones(_EAGER_DIM)
def preprocess(input_ids, input_embeds, **info):
updates = {"mtp_inputs": (torch.zeros(_EAGER_DIM), text_step)} if input_ids.shape[0] == 1 else {}
return input_ids, input_embeds, updates
state.model.preprocess = preprocess
batch = _EagerBatch([3, 3, 1])
model_inputs = {"input_ids": torch.zeros(7, dtype=torch.long), "inputs_embeds": torch.zeros((7, _EAGER_DIM))}
req_states = SimpleNamespace(
prompt_len=np.array([10, 10, 10], dtype=np.int32),
num_computed_tokens=np.array([0, 7, 10], dtype=np.int32),
)
state.run_preprocess(batch, model_inputs, req_states)
recorded_batch, entries, _ids = state._eager_rows
assert recorded_batch is batch
# A non-final prefill chunk samples nothing that is kept.
assert entries == [(1, 1, "final", True), (2, 2, "decode", False)]
assert model_inputs["inputs_embeds"][6].tolist() == [5.0] * _EAGER_DIM
assert state.model.mtp_calls == []
@pytest.mark.parametrize("first_was_valid", [False, True])
def test_first_audio_requirement_survives_eos_only_if_audio_was_queued(first_was_valid):
state = _make_eager_state()
_fill_buffers(state, "r0")
state._first_audio_requests.add("r0")
state._eager_state._first_audio_valid = torch.tensor([first_was_valid, False, False, False])
batch = _EagerBatch([1])
state._eager_rows = (batch, [(0, 0, "r0", False)], torch.tensor([5]))
outputs = _eager_outputs(1)
outputs["meta"]["first_audio"] = torch.zeros(1, dtype=torch.bool)
state.run_eager_mtp(batch, torch.zeros(1, _EAGER_DIM), torch.tensor([[_EOS]]), outputs)
assert outputs["meta"]["first_audio"].tolist() == [first_was_valid]
@pytest.mark.parametrize("accepted", [[], ["r1"], ["r0", "r1"]])
def test_first_audio_marker_requires_accepted_delivery(monkeypatch, accepted):
from contextlib import nullcontext
state = _make_eager_state()
_fill_buffers(state, "r0", "r1")
state.model.first_frame_decoder = SimpleNamespace(
sample_rate=24000, decode=lambda codes: torch.ones(codes.shape[0], 2)
)
state._first_audio_stream = SimpleNamespace(wait_stream=lambda stream: None)
state._first_audio_sender = SimpleNamespace(submit=lambda ids, pcm, sr, valid: accepted)
monkeypatch.setattr(torch.cuda, "current_stream", lambda device: None)
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
monkeypatch.setattr(torch.Tensor, "record_stream", lambda tensor, stream: None)
batch = _EagerBatch([1, 1])
state._eager_rows = (batch, [(0, 0, "r0", True), (1, 1, "r1", True)], torch.zeros(2, dtype=torch.long))
outputs = _eager_outputs(2)
outputs["meta"]["first_audio"] = torch.zeros(2, dtype=torch.bool)
state.run_eager_mtp(batch, torch.zeros(2, _EAGER_DIM), torch.tensor([[7], [8]]), outputs)
assert state._first_audio_requests == set(accepted)
# A missing route must leave the normal codec path responsible for frame 0;
# otherwise it skips the frame and the orchestrator waits forever for it.
assert outputs["meta"]["first_audio"].tolist() == ["r0" in accepted, "r1" in accepted]