[Model] Optimize Qwen3-TTS streaming throughput and first audio on CUDA (#8163)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
55 files changed
+4222
-230
No files matched your search
@@ -166,7 +166,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
assert category_counts == {
|
||||
EnvironmentVariableCategory.PUBLIC_OMNI: 30,
|
||||
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
|
||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 29,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 58,
|
||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
|
||||
EnvironmentVariableCategory.INTERNAL: 2,
|
||||
|
||||
@@ -886,6 +886,7 @@ def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
|
||||
|
||||
def test_runtime_config_fields_match_structured_runtime_scope():
|
||||
assert {f.name for f in fields(OmniStageRuntimeConfig)} == {
|
||||
"cuda_mps",
|
||||
"additional_config",
|
||||
"distributed_executor_backend",
|
||||
"worker_cls",
|
||||
@@ -2051,6 +2052,28 @@ def test_async_chunk_rejects_mismatched_connector_edge(disabled_stage, builder):
|
||||
builder(pipeline, user_deploy_config=deploy)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scope", ["pipeline", "stage"])
|
||||
@pytest.mark.parametrize("from_yaml", [False, True])
|
||||
@pytest.mark.parametrize("builder", [merge_pipeline_deploy, VllmOmniConfig.from_pipeline_config])
|
||||
def test_async_chunk_rejects_quoted_false_before_selecting_processors(tmp_path, scope, from_yaml, builder):
|
||||
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
|
||||
if from_yaml:
|
||||
path = tmp_path / "quoted_false.yaml"
|
||||
path.write_text(
|
||||
'async_chunk: "false"\n' if scope == "pipeline" else 'stages:\n - stage_id: 0\n async_chunk: "false"\n'
|
||||
)
|
||||
deploy = load_deploy_config(path)
|
||||
elif scope == "pipeline":
|
||||
deploy = DeployConfig(async_chunk="false")
|
||||
else:
|
||||
deploy = DeployConfig(stages=[StageDeployConfig(stage_id=0, async_chunk="false")])
|
||||
with pytest.raises(ValueError, match="async_chunk must be a boolean"):
|
||||
if builder is merge_pipeline_deploy:
|
||||
builder(pipeline, deploy)
|
||||
else:
|
||||
builder(pipeline, user_deploy_config=deploy)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("explicit", [False, True])
|
||||
def test_diffusion_quantization_origin_survives_projection_and_transport(monkeypatch, explicit):
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
|
||||
@@ -2169,3 +2192,31 @@ def test_structured_diffusion_stage_keeps_shared_globals_outside_diffusion():
|
||||
def test_structured_diffusion_stage_rejects_explicit_shared_engine_field(field_name, config_kwargs):
|
||||
with pytest.raises(ValueError, match=rf"stage 0.*{field_name}"):
|
||||
_build_single_diffusion_config(**config_kwargs)
|
||||
|
||||
|
||||
def test_mps_is_explicit_only_in_experimental_single_gpu_profile():
|
||||
assert not load_deploy_config(_DEPLOY_DIR / "qwen3_tts.yaml").cuda_mps
|
||||
deploy = load_deploy_config(_DEPLOY_DIR / "qwen3_tts_high_concurrency_mrv2_single_gpu.yaml")
|
||||
assert deploy.cuda_mps
|
||||
pipeline = _resolve_pipeline_or_skip("qwen3_tts")
|
||||
stages = merge_pipeline_deploy(pipeline, deploy)
|
||||
assert all(stage.yaml_runtime["cuda_mps"] is True for stage in stages)
|
||||
|
||||
|
||||
def test_mps_config_rejects_string_boolean(tmp_path):
|
||||
path = tmp_path / "deploy.yaml"
|
||||
path.write_text('cuda_mps: "false"\n')
|
||||
with pytest.raises(ValueError, match="cuda_mps must be a boolean"):
|
||||
load_deploy_config(path)
|
||||
|
||||
|
||||
def test_mps_stays_in_runtime_instead_of_engine_arguments():
|
||||
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
|
||||
|
||||
config = _from_pipeline_key(
|
||||
"qwen3_tts", deploy_config_path=str(_DEPLOY_DIR / "qwen3_tts_high_concurrency_mrv2_single_gpu.yaml")
|
||||
)
|
||||
for stage in config.stage_configs:
|
||||
assert stage.runtime_config.cuda_mps
|
||||
args = build_engine_args_dict_from_omni_stage_config(stage, model="test-model")
|
||||
assert "cuda_mps" not in args
|
||||
@@ -66,6 +66,53 @@ default_tts_server_params = [
|
||||
)
|
||||
]
|
||||
|
||||
# Exercise the throughput path in the existing L4 lane with smaller capacities.
|
||||
# The production H200 profile retains its own admission and graph buckets.
|
||||
_FAST_PATH_STAGE_CONFIG = modify_stage_config(
|
||||
get_deploy_config_path("qwen3_tts_high_concurrency_mrv2_single_gpu.yaml"),
|
||||
updates={
|
||||
# The temporary overlay lives outside deploy/, so anchor its parent.
|
||||
"base_config": get_deploy_config_path("qwen3_tts_high_concurrency_mrv2.yaml"),
|
||||
"cuda_mps": False,
|
||||
"connectors.connector_of_shared_memory.extra.decode_batch_max_size": 2,
|
||||
"connectors.connector_of_shared_memory.extra.decode_cudagraph_batch_sizes": [1, 2],
|
||||
"stages": {
|
||||
0: {
|
||||
"max_num_seqs": 4,
|
||||
"max_num_batched_tokens": 128,
|
||||
"compilation_config.cudagraph_capture_sizes": [1, 2, 4, 8, 16, 32, 64, 128],
|
||||
"compilation_config.max_cudagraph_capture_size": 128,
|
||||
},
|
||||
1: {"max_num_seqs": 4},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
[OmniServerParams(model=MODEL, stage_config_path=_FAST_PATH_STAGE_CONFIG, server_args=["--trust-remote-code"])],
|
||||
indirect=True,
|
||||
)
|
||||
def test_cached_predictor_streaming_audio(omni_server, online_client) -> None:
|
||||
"""Cover the optimized predictor, first audio and codec path with real weights at L3."""
|
||||
online_client.send_audio_speech_request(
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"input": "The quick brown fox jumps over the lazy dog.",
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "wav",
|
||||
"task_type": "CustomVoice",
|
||||
"voice": "vivian",
|
||||
},
|
||||
request_num=4,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
|
||||
@@ -10,6 +10,7 @@ import threading
|
||||
import time
|
||||
import types
|
||||
from dataclasses import dataclass, field
|
||||
from subprocess import CompletedProcess
|
||||
|
||||
import pytest
|
||||
from omegaconf import OmegaConf
|
||||
@@ -70,6 +71,49 @@ def test_stage_runtime_env_accepts_typed_runtime_config(monkeypatch):
|
||||
assert env_key not in os.environ
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_key,invalid_value",
|
||||
[("INVALID=ENV", "value"), ("INVALID\0ENV", "value"), ("INVALID_ENV", "value\0")],
|
||||
ids=["equals-in-key", "nul-in-key", "nul-in-value"],
|
||||
)
|
||||
def test_stage_runtime_env_restores_partial_application(monkeypatch, invalid_key, invalid_value):
|
||||
existing_key = "VLLM_OMNI_TEST_EXISTING_STAGE_ENV"
|
||||
new_key = "VLLM_OMNI_TEST_NEW_STAGE_ENV"
|
||||
monkeypatch.setenv(existing_key, "original")
|
||||
monkeypatch.delenv(new_key, raising=False)
|
||||
runtime_config = OmniStageRuntimeConfig(
|
||||
env={existing_key: "overridden", new_key: "temporary", invalid_key: invalid_value}
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
with stage_runtime_env(0, runtime_config):
|
||||
pytest.fail("Invalid environment must prevent stage launch")
|
||||
|
||||
assert os.environ[existing_key] == "original"
|
||||
assert new_key not in os.environ
|
||||
|
||||
|
||||
def test_stage_runtime_env_restores_after_launch_error(monkeypatch):
|
||||
env_key = "VLLM_OMNI_TEST_STAGE_ENV_LAUNCH_ERROR"
|
||||
monkeypatch.setenv(env_key, "original")
|
||||
|
||||
with pytest.raises(RuntimeError, match="stage launch failed"):
|
||||
with stage_runtime_env(0, OmniStageRuntimeConfig(env={env_key: "temporary"})):
|
||||
assert os.environ[env_key] == "temporary"
|
||||
raise RuntimeError("stage launch failed")
|
||||
|
||||
assert os.environ[env_key] == "original"
|
||||
|
||||
|
||||
def test_stage_runtime_env_restores_first_value_for_normalized_keys(monkeypatch):
|
||||
monkeypatch.setenv("123", "original")
|
||||
|
||||
with stage_runtime_env(0, {"env": {123: "first", "123": "second"}}):
|
||||
assert os.environ["123"] == "second"
|
||||
|
||||
assert os.environ["123"] == "original"
|
||||
|
||||
|
||||
def test_orchestrator_startup_timeout_warns_how_to_raise_limits(monkeypatch):
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.orchestrator_thread = types.SimpleNamespace(is_alive=lambda: True)
|
||||
@@ -1019,6 +1063,63 @@ def test_stage_runtime_launches_shared_engines_with_per_client_addresses(monkeyp
|
||||
assert os.environ.get("VLLM_OMNI_TEST_STAGE_RUNTIME_ENV") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_count", [1, 2])
|
||||
def test_stage_launch_uses_configured_mps_pipe_and_rejects_conflicts(monkeypatch, client_count):
|
||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||
from vllm_omni.engine import cuda_mps
|
||||
|
||||
runtime = _make_stage_runtime()
|
||||
plans = [_make_llm_plan(stage_id, stage_id=stage_id, vllm_config=_FakeVllmConfig()) for stage_id in (0, 1)]
|
||||
for stage_id, plan in enumerate(plans):
|
||||
plan.replicas[0].engine_args_dict = {}
|
||||
plan.replicas[0].metadata.runtime_cfg = OmniStageRuntimeConfig(
|
||||
cuda_mps=True,
|
||||
devices="0",
|
||||
env={"CUDA_MPS_PIPE_DIRECTORY": f"/operator/stage-{stage_id}"},
|
||||
)
|
||||
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/parent")
|
||||
monkeypatch.setattr(runtime, "_prepare_stage_plans", lambda: plans)
|
||||
monkeypatch.setattr(runtime, "_resolve_replica_physical_devices", lambda *_: "0")
|
||||
monkeypatch.setattr(runtime_mod, "acquire_device_locks", lambda *_args, **_kwargs: [])
|
||||
monkeypatch.setattr(runtime_mod, "physical_gpu_uuid", lambda _: "GPU-example")
|
||||
monkeypatch.setattr(runtime_mod.current_omni_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(cuda_mps.shutil, "which", lambda _: "/bin/mps-control")
|
||||
controls = []
|
||||
|
||||
def run_control(args, **kwargs):
|
||||
controls.append(kwargs)
|
||||
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
|
||||
assert kwargs["env"]["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/stage-0"
|
||||
return CompletedProcess(args, 0, stdout="")
|
||||
|
||||
monkeypatch.setattr(cuda_mps.subprocess, "run", run_control)
|
||||
launched = []
|
||||
|
||||
@contextlib.contextmanager
|
||||
def launch(**kwargs):
|
||||
launched.append(kwargs["stage_id"])
|
||||
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/stage-0"
|
||||
assert kwargs["stage_visible_devices"] == "GPU-example"
|
||||
yield StageReplicaResources(
|
||||
addresses=EngineZmqAddresses(
|
||||
inputs=[f"ipc://input-{i}" for i in range(client_count)],
|
||||
outputs=[f"ipc://output-{i}" for i in range(client_count)],
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(runtime_mod, "launch_stage_replica", launch)
|
||||
try:
|
||||
with pytest.raises(ValueError, match="Conflicting CUDA_MPS_PIPE_DIRECTORY"):
|
||||
with runtime.launch_stage_engines(client_count):
|
||||
pytest.fail("Conflicting colocated MPS policies must prevent launch")
|
||||
assert launched == [0]
|
||||
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
assert len(controls) == 1
|
||||
assert controls[0]["input"] == "get_server_list\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_count", [1, 2])
|
||||
def test_stage_runtime_overlapping_devices_acquire_real_locks_once(monkeypatch, tmp_path, client_count):
|
||||
import fcntl
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.config.omni_config import OmniStageRuntimeConfig
|
||||
from vllm_omni.engine import cuda_mps
|
||||
from vllm_omni.engine.stage_runtime import StageRuntime
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def controls(monkeypatch):
|
||||
monkeypatch.delenv("CUDA_MPS_PIPE_DIRECTORY", raising=False)
|
||||
monkeypatch.setattr(cuda_mps.shutil, "which", lambda name: "/bin/mps-control")
|
||||
run = Mock(return_value=SimpleNamespace(stdout=""))
|
||||
monkeypatch.setattr(cuda_mps.subprocess, "run", run)
|
||||
return run
|
||||
|
||||
|
||||
def test_private_server_has_scoped_environment_and_idempotent_close(controls):
|
||||
original = dict(os.environ)
|
||||
server = cuda_mps.CudaMPSServer("GPU-example")
|
||||
pipe = Path(server.env["CUDA_MPS_PIPE_DIRECTORY"])
|
||||
assert pipe.is_dir()
|
||||
assert controls.call_args_list[0].args[0] == ["/bin/mps-control", "-d"]
|
||||
assert controls.call_args_list[0].kwargs["env"]["CUDA_VISIBLE_DEVICES"] == "GPU-example"
|
||||
assert dict(os.environ) == original
|
||||
server.close()
|
||||
count = controls.call_count
|
||||
server.close()
|
||||
assert controls.call_count == count
|
||||
assert controls.call_args.kwargs["input"] == "quit\n"
|
||||
assert not pipe.exists()
|
||||
|
||||
|
||||
def test_operator_daemon_is_checked_but_never_stopped(controls, monkeypatch):
|
||||
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/mps")
|
||||
server = cuda_mps.CudaMPSServer("GPU-example")
|
||||
server.close()
|
||||
assert controls.call_count == 1
|
||||
assert controls.call_args.kwargs["input"] == "get_server_list\n"
|
||||
|
||||
|
||||
def test_explicit_pipe_overrides_parent_without_taking_ownership(controls, monkeypatch):
|
||||
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/parent")
|
||||
server = cuda_mps.CudaMPSServer("GPU-example", pipe_directory="/operator/stage")
|
||||
|
||||
assert controls.call_args.kwargs["env"]["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/stage"
|
||||
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
|
||||
server.close()
|
||||
assert controls.call_count == 1
|
||||
|
||||
|
||||
def test_explicit_empty_pipe_selects_private_daemon(controls, monkeypatch):
|
||||
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/parent")
|
||||
server = cuda_mps.CudaMPSServer("GPU-example", pipe_directory="")
|
||||
pipe = server.env["CUDA_MPS_PIPE_DIRECTORY"]
|
||||
try:
|
||||
assert pipe != "/operator/parent"
|
||||
assert controls.call_args_list[0].args[0] == ["/bin/mps-control", "-d"]
|
||||
assert controls.call_args_list[0].kwargs["env"]["CUDA_MPS_PIPE_DIRECTORY"] == pipe
|
||||
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
|
||||
finally:
|
||||
server.close()
|
||||
assert controls.call_args.kwargs["input"] == "quit\n"
|
||||
assert controls.call_args.kwargs["env"]["CUDA_MPS_PIPE_DIRECTORY"] == pipe
|
||||
assert not Path(pipe).exists()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mps_runtime(monkeypatch):
|
||||
from vllm_omni.engine import stage_runtime
|
||||
|
||||
runtime = StageRuntime.__new__(StageRuntime)
|
||||
runtime._mps_servers = {}
|
||||
monkeypatch.setattr(stage_runtime, "physical_gpu_uuid", lambda device: "GPU-example")
|
||||
monkeypatch.setattr(stage_runtime.current_omni_platform, "is_cuda", lambda: True)
|
||||
yield runtime
|
||||
runtime._close_mps_servers()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pipe_directory", ["/operator/stage", ""])
|
||||
def test_runtime_shares_first_stage_pipe_policy(controls, monkeypatch, mps_runtime, pipe_directory):
|
||||
monkeypatch.setenv("CUDA_MPS_PIPE_DIRECTORY", "/operator/parent")
|
||||
config = OmniStageRuntimeConfig(cuda_mps=True, env={"CUDA_MPS_PIPE_DIRECTORY": pipe_directory})
|
||||
first_env = mps_runtime._mps_environment("0", config)
|
||||
selected_pipe = first_env["CUDA_MPS_PIPE_DIRECTORY"]
|
||||
if pipe_directory:
|
||||
assert selected_pipe == pipe_directory
|
||||
else:
|
||||
assert selected_pipe != "/operator/parent"
|
||||
assert Path(selected_pipe).is_dir()
|
||||
call_count = controls.call_count
|
||||
|
||||
assert mps_runtime._mps_environment("0", {"cuda_mps": True}) is first_env
|
||||
assert mps_runtime._mps_environment("0", config) is first_env
|
||||
assert controls.call_count == call_count
|
||||
assert os.environ["CUDA_MPS_PIPE_DIRECTORY"] == "/operator/parent"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"first_pipe,conflicting_pipe",
|
||||
[("/operator/first", "/operator/other"), ("/operator/first", ""), ("", "/operator/other")],
|
||||
)
|
||||
def test_runtime_rejects_conflicting_explicit_stage_pipes(controls, mps_runtime, first_pipe, conflicting_pipe):
|
||||
mps_runtime._mps_environment("0", {"cuda_mps": True, "env": {"CUDA_MPS_PIPE_DIRECTORY": first_pipe}})
|
||||
call_count = controls.call_count
|
||||
|
||||
with pytest.raises(ValueError, match="Conflicting CUDA_MPS_PIPE_DIRECTORY"):
|
||||
mps_runtime._mps_environment("0", {"cuda_mps": True, "env": {"CUDA_MPS_PIPE_DIRECTORY": conflicting_pipe}})
|
||||
|
||||
assert controls.call_count == call_count
|
||||
|
||||
|
||||
def test_startup_failure_closes_only_the_private_daemon(controls):
|
||||
controls.side_effect = [SimpleNamespace(stdout=""), RuntimeError("health failed"), SimpleNamespace(stdout="")]
|
||||
with pytest.raises(RuntimeError, match="health failed"):
|
||||
cuda_mps.CudaMPSServer("GPU-example")
|
||||
assert controls.call_args.kwargs["input"] == "quit\n"
|
||||
|
||||
|
||||
def test_runtime_opt_in_and_shared_gpu_ownership(monkeypatch):
|
||||
from vllm_omni.engine import stage_runtime
|
||||
|
||||
runtime = StageRuntime.__new__(StageRuntime)
|
||||
runtime._mps_servers = {}
|
||||
create = Mock(return_value=SimpleNamespace(env={"CUDA_VISIBLE_DEVICES": "GPU-example"}, close=Mock()))
|
||||
monkeypatch.setattr(stage_runtime, "CudaMPSServer", create)
|
||||
monkeypatch.setattr(stage_runtime, "physical_gpu_uuid", lambda device: "GPU-example")
|
||||
monkeypatch.setattr(stage_runtime, "current_omni_platform", SimpleNamespace(is_cuda=lambda: True))
|
||||
assert runtime._mps_environment("0", {}) == {}
|
||||
assert create.call_count == 0
|
||||
for _ in range(2):
|
||||
assert runtime._mps_environment("0", {"cuda_mps": True}) == {"CUDA_VISIBLE_DEVICES": "GPU-example"}
|
||||
assert create.call_count == 1
|
||||
runtime._close_mps_servers()
|
||||
runtime._close_mps_servers()
|
||||
create.return_value.close.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("devices", [None, "0,1"])
|
||||
def test_runtime_rejects_ambiguous_placement(devices, monkeypatch):
|
||||
from vllm_omni.engine import stage_runtime
|
||||
|
||||
runtime = StageRuntime.__new__(StageRuntime)
|
||||
monkeypatch.setattr(stage_runtime, "current_omni_platform", SimpleNamespace(is_cuda=lambda: True))
|
||||
with pytest.raises(ValueError, match="exactly one explicit GPU"):
|
||||
runtime._mps_environment(devices, {"cuda_mps": True})
|
||||
|
||||
|
||||
def test_failed_shutdown_preserves_socket_for_retry(controls):
|
||||
server = cuda_mps.CudaMPSServer("GPU-example")
|
||||
pipe = Path(server.env["CUDA_MPS_PIPE_DIRECTORY"])
|
||||
controls.side_effect = RuntimeError("quit failed")
|
||||
with pytest.raises(RuntimeError, match="quit failed"):
|
||||
server.close()
|
||||
assert pipe.is_dir()
|
||||
controls.side_effect = None
|
||||
server.close()
|
||||
assert not pipe.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("launch_mode,stage_type", [("remote", "llm"), ("local", "diffusion")])
|
||||
def test_mps_rejects_unsupported_stages(launch_mode, stage_type):
|
||||
runtime = StageRuntime.__new__(StageRuntime)
|
||||
metadata = SimpleNamespace(runtime_cfg={"cuda_mps": True}, stage_type=stage_type)
|
||||
stage_cfg = SimpleNamespace(runtime=metadata.runtime_cfg)
|
||||
plans = [
|
||||
SimpleNamespace(replicas=[SimpleNamespace(metadata=metadata, stage_cfg=stage_cfg, launch_mode=launch_mode)])
|
||||
]
|
||||
with pytest.raises(ValueError, match="local EngineCore"):
|
||||
runtime._validate_mps_topology(plans)
|
||||
metadata.runtime_cfg["cuda_mps"] = False
|
||||
runtime._validate_mps_topology(plans)
|
||||
|
||||
|
||||
def test_mps_rejects_parallel_initialization():
|
||||
runtime = StageRuntime.__new__(StageRuntime)
|
||||
runtime._parallel_stage_init = True
|
||||
replica = SimpleNamespace(
|
||||
metadata=SimpleNamespace(stage_type="llm"),
|
||||
stage_cfg=SimpleNamespace(runtime={"cuda_mps": True}),
|
||||
launch_mode="local",
|
||||
)
|
||||
with pytest.raises(ValueError, match="parallel_stage_init=false"):
|
||||
runtime._validate_mps_topology([SimpleNamespace(replicas=[replica])])
|
||||
|
||||
|
||||
def test_failed_launch_command_still_checks_private_daemon_cleanup(controls):
|
||||
controls.side_effect = [RuntimeError("launch failed after fork"), SimpleNamespace(stdout="")]
|
||||
with pytest.raises(RuntimeError, match="launch failed after fork"):
|
||||
cuda_mps.CudaMPSServer("GPU-example")
|
||||
assert controls.call_args.kwargs["input"] == "quit\n"
|
||||
@@ -425,3 +425,11 @@ class TestSerializeDeserializePayload:
|
||||
restored = deserialize_payload(wire)
|
||||
|
||||
assert restored["meta"]["minimax_h3_prepared_reference_videos"] == descriptor
|
||||
|
||||
|
||||
def test_first_audio_metadata_round_trips_as_a_tensor():
|
||||
from vllm_omni.data_entry_keys import to_dict, to_struct
|
||||
|
||||
payload = {"meta": {"first_audio": torch.tensor(True)}}
|
||||
restored = to_dict(to_struct(payload))
|
||||
assert restored["meta"]["first_audio"].item() is True
|
||||
@@ -0,0 +1,345 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
from vllm_omni.engine import OmniEngineCoreOutput, OmniEngineCoreOutputs
|
||||
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
|
||||
from vllm_omni.engine.messages import ErrorMessage, OutputMessage
|
||||
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
|
||||
from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _CodecPool:
|
||||
final_output = True
|
||||
|
||||
def __init__(self, output_kind, registered, replica_id):
|
||||
self.output_processor = MultimodalOutputProcessor(
|
||||
tokenizer=None, log_stats=False, engine_core_output_type="audio"
|
||||
)
|
||||
self.output_kind = output_kind
|
||||
self.replica_id = replica_id
|
||||
self.bound_replica = None
|
||||
if registered:
|
||||
self.register()
|
||||
|
||||
def register(self):
|
||||
self.bound_replica = self.replica_id
|
||||
self.output_processor.request_states["r"] = OmniRequestState(
|
||||
request_id="r",
|
||||
external_req_id="r",
|
||||
parent_req=None,
|
||||
request_index=0,
|
||||
lora_request=None,
|
||||
prompt=None,
|
||||
prompt_token_ids=[0],
|
||||
prompt_embeds=None,
|
||||
logprobs_processor=None,
|
||||
detokenizer=None,
|
||||
max_tokens_param=None,
|
||||
arrival_time=0.0,
|
||||
queue=None,
|
||||
log_stats=False,
|
||||
stream_interval=1,
|
||||
output_kind=self.output_kind,
|
||||
)
|
||||
self.output_processor.external_req_ids["r"].append("r")
|
||||
|
||||
def get_bound_replica_id(self, request_id):
|
||||
return self.bound_replica
|
||||
|
||||
async def process_llm_raw_outputs(self, replica_id, raw, **kwargs):
|
||||
assert replica_id == self.replica_id
|
||||
return self.output_processor.process_outputs(raw.outputs, raw.timestamp).request_outputs
|
||||
|
||||
|
||||
def _orchestrator(output_kind=RequestOutputKind.DELTA, *, registered=True, replica_id=2, final_stage_id=1):
|
||||
obj = Orchestrator.__new__(Orchestrator)
|
||||
obj.stage_pools = [SimpleNamespace(final_output=False), _CodecPool(output_kind, registered, replica_id)]
|
||||
obj.request_states = {"r": OrchestratorRequestState(request_id="r", final_stage_id=final_stage_id)}
|
||||
obj.output_async_queue = asyncio.Queue()
|
||||
obj._cfg_tracker = CfgCompanionTracker()
|
||||
obj._pd_kv_params = {}
|
||||
obj._running_counter = None
|
||||
obj._abort_request_ids = AsyncMock(return_value=[])
|
||||
obj._release_request_bindings = Mock()
|
||||
obj._finish_raw_terminal_requests = AsyncMock()
|
||||
|
||||
async def process(stage, replica, raw, terminals):
|
||||
await obj._route_upstream_first_audio(stage, replica, raw)
|
||||
return await obj.stage_pools[stage].process_llm_raw_outputs(replica, raw)
|
||||
|
||||
async def route(stage, replica, outputs):
|
||||
for output in outputs:
|
||||
await obj.output_async_queue.put(
|
||||
OutputMessage(
|
||||
request_id=output.request_id,
|
||||
stage_id=stage,
|
||||
replica_id=replica,
|
||||
engine_outputs=output,
|
||||
metrics=None,
|
||||
finished=output.finished,
|
||||
)
|
||||
)
|
||||
|
||||
obj._process_llm_stage_outputs = process
|
||||
obj._handle_processed_outputs = route
|
||||
return obj
|
||||
|
||||
|
||||
def _raw(*, first=False, required=False, terminal=False, samples=(1, 1, 1, 1)):
|
||||
return OmniEngineCoreOutputs(
|
||||
outputs=[
|
||||
OmniEngineCoreOutput(
|
||||
request_id="r",
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.STOP if terminal else None,
|
||||
multimodal_output={
|
||||
"model_outputs": torch.tensor(samples, dtype=torch.float32),
|
||||
"sr": torch.tensor(24000),
|
||||
"_omni_first_audio": torch.tensor(first),
|
||||
"_omni_first_audio_required": torch.tensor(required),
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def _codec_output(obj, raw):
|
||||
replica = obj.stage_pools[1].replica_id
|
||||
processed = await obj._process_llm_stage_outputs(1, replica, raw, set())
|
||||
await obj._handle_processed_outputs(1, replica, processed)
|
||||
|
||||
|
||||
def _messages(obj):
|
||||
messages = []
|
||||
while not obj.output_async_queue.empty():
|
||||
messages.append(obj.output_async_queue.get_nowait())
|
||||
return messages
|
||||
|
||||
|
||||
def _audio(output):
|
||||
audio = output.outputs[0].multimodal_output["audio"]
|
||||
return torch.cat(audio) if isinstance(audio, list) else audio
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", list(RequestOutputKind))
|
||||
@pytest.mark.parametrize("codec_overtook", [False, True])
|
||||
async def test_first_audio_obeys_codec_output_kind(kind, codec_overtook):
|
||||
obj = _orchestrator(kind)
|
||||
suffix = _raw(required=True, samples=(20, 21))
|
||||
terminal = _raw(terminal=True, samples=(30, 31))
|
||||
if codec_overtook:
|
||||
await _codec_output(obj, suffix)
|
||||
await _codec_output(obj, terminal)
|
||||
assert obj.output_async_queue.empty()
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
|
||||
if kind == RequestOutputKind.FINAL_ONLY and not codec_overtook:
|
||||
assert obj.output_async_queue.empty()
|
||||
if not codec_overtook:
|
||||
await _codec_output(obj, suffix)
|
||||
await _codec_output(obj, terminal)
|
||||
messages = _messages(obj)
|
||||
# The source replica is zero; output ownership belongs to codec replica two.
|
||||
assert all((message.stage_id, message.replica_id) == (1, 2) for message in messages)
|
||||
actual = [_audio(message.engine_outputs).tolist() for message in messages]
|
||||
expected = {
|
||||
RequestOutputKind.DELTA: [[10, 11], [20, 21], [30, 31]],
|
||||
RequestOutputKind.CUMULATIVE: [[10, 11], [10, 11, 20, 21], [10, 11, 20, 21, 30, 31]],
|
||||
RequestOutputKind.FINAL_ONLY: [[10, 11, 20, 21, 30, 31]],
|
||||
}
|
||||
assert actual == expected[kind]
|
||||
assert messages[-1].finished
|
||||
assert obj.request_states["r"].pending_first_audio_outputs == []
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
|
||||
assert obj.output_async_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_audio_waits_for_codec_registration_and_ignores_duplicates():
|
||||
obj = _orchestrator(registered=False)
|
||||
req_state = obj.request_states["r"]
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(90, 91)))
|
||||
assert obj.output_async_queue.empty()
|
||||
assert not req_state.upstream_first_audio
|
||||
assert req_state.pending_upstream_first_audio is not None
|
||||
obj.stage_pools[1].register()
|
||||
await obj._flush_upstream_first_audio(req_state)
|
||||
messages = _messages(obj)
|
||||
assert len(messages) == 1
|
||||
assert _audio(messages[0].engine_outputs).tolist() == [10, 11]
|
||||
assert req_state.upstream_first_audio
|
||||
assert req_state.pending_upstream_first_audio is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("processing_failed", [False, True])
|
||||
async def test_prewarm_flushes_first_audio_after_successful_submission(mocker, processing_failed):
|
||||
obj = _orchestrator(registered=False, final_stage_id=2 if processing_failed else 1)
|
||||
req_state = obj.request_states["r"]
|
||||
req_state.sampling_params_list = [None] * (req_state.final_stage_id + 1)
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
|
||||
pool = obj.stage_pools[1]
|
||||
pool.stage_type = "llm"
|
||||
pool.stage_vllm_config = SimpleNamespace(model_config=SimpleNamespace(hf_config=None))
|
||||
obj.stage_pools[0].get_bound_replica_id = lambda request_id: 0
|
||||
obj._stage_receives_async_chunks = lambda stage: True
|
||||
obj._build_payload_sender_info = Mock(return_value={})
|
||||
obj._on_stage_submitted = Mock()
|
||||
obj._emit_tx_edge = Mock()
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.orchestrator.build_engine_core_request_from_tokens",
|
||||
return_value=SimpleNamespace(request_id="r"),
|
||||
)
|
||||
|
||||
async def submit(*args, **kwargs):
|
||||
pool.register()
|
||||
|
||||
pool.submit_initial = submit
|
||||
if processing_failed:
|
||||
pool.process_llm_raw_outputs = AsyncMock(side_effect=RuntimeError("processing failed"))
|
||||
later_pool = SimpleNamespace(submit_initial=AsyncMock())
|
||||
obj.stage_pools.append(later_pool)
|
||||
submitted = await obj._prewarm_async_chunk_stages("r", SimpleNamespace(prompt_token_ids=[0]), req_state)
|
||||
assert submitted is not processing_failed
|
||||
messages = _messages(obj)
|
||||
assert len(messages) == 1
|
||||
if processing_failed:
|
||||
assert isinstance(messages[0], ErrorMessage)
|
||||
later_pool.submit_initial.assert_not_called()
|
||||
assert "r" not in obj.request_states
|
||||
else:
|
||||
assert _audio(messages[0].engine_outputs).tolist() == [10, 11]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_first_audio_is_discarded_on_cancellation():
|
||||
obj = _orchestrator(registered=False)
|
||||
req_state = obj.request_states["r"]
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
|
||||
await obj._cleanup_request_ids(["r"], abort=True)
|
||||
obj.stage_pools[1].register()
|
||||
await obj._flush_upstream_first_audio(req_state)
|
||||
assert obj.output_async_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancellation_while_routing_first_audio_does_not_release_codec_outputs():
|
||||
obj = _orchestrator()
|
||||
await obj._route_upstream_first_audio(1, 2, _raw(required=True, terminal=True))
|
||||
|
||||
async def cancel(stage, replica, outputs):
|
||||
await obj._cleanup_request_ids(["r"], abort=True)
|
||||
|
||||
obj._handle_processed_outputs = cancel
|
||||
obj._process_llm_stage_outputs = AsyncMock()
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
|
||||
obj._process_llm_stage_outputs.assert_not_called()
|
||||
assert obj.output_async_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_ending_at_source_does_not_emit_first_audio():
|
||||
obj = _orchestrator(final_stage_id=0)
|
||||
raw = _raw(first=True)
|
||||
await obj._route_upstream_first_audio(0, 0, raw)
|
||||
assert not raw.outputs and obj.output_async_queue.empty()
|
||||
assert obj.request_states["r"].pending_upstream_first_audio is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_later_aligner_receives_full_codec_waveform():
|
||||
from vllm_omni.model_executor.stage_input_processors.forced_aligner import code2wav2aligner
|
||||
|
||||
obj = _orchestrator(RequestOutputKind.CUMULATIVE, final_stage_id=2)
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
|
||||
await _codec_output(obj, _raw(required=True, terminal=True, samples=(20, 21)))
|
||||
messages = _messages(obj)
|
||||
assert all(message.stage_id == 1 for message in messages)
|
||||
inputs = code2wav2aligner(
|
||||
[messages[-1].engine_outputs], prompt={"additional_information": {"text": ["Hello world"]}}
|
||||
)
|
||||
assert inputs[0]["multi_modal_data"]["audio"][0].tolist() == [10, 11, 20, 21]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_audio_processing_failure_fails_only_its_request():
|
||||
obj = _orchestrator()
|
||||
obj.request_states["healthy"] = OrchestratorRequestState(request_id="healthy", final_stage_id=1)
|
||||
obj.stage_pools[1].process_llm_raw_outputs = AsyncMock(side_effect=RuntimeError("processing failed"))
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
|
||||
error = obj.output_async_queue.get_nowait()
|
||||
assert isinstance(error, ErrorMessage)
|
||||
assert error.request_id == "r" and error.stage_id == 1
|
||||
assert "processing failed" in error.error
|
||||
assert set(obj.request_states) == {"healthy"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unmarked_upstream_audio_and_regular_codec_outputs_are_untouched():
|
||||
obj = _orchestrator()
|
||||
for stage in (0, 1):
|
||||
raw = _raw()
|
||||
await obj._route_upstream_first_audio(stage, 0, raw)
|
||||
assert len(raw.outputs) == 1
|
||||
assert torch.equal(raw.outputs[0].multimodal_output["model_outputs"], torch.ones(4))
|
||||
assert obj.output_async_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_request_does_not_receive_late_first_audio():
|
||||
obj = _orchestrator()
|
||||
obj.request_states.clear()
|
||||
raw = _raw(first=True)
|
||||
await obj._route_upstream_first_audio(0, 0, raw)
|
||||
assert not raw.outputs and obj.output_async_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_audio_supports_both_speech_and_chat_consumers():
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
|
||||
obj = _orchestrator()
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True, samples=(10, 11)))
|
||||
result = obj.output_async_queue.get_nowait().engine_outputs
|
||||
assert not result.finished
|
||||
assert _audio(result).tolist() == [10, 11]
|
||||
payload, key = OmniOpenAIServingSpeech._extract_audio_output(result)
|
||||
assert payload is not None and key is not None
|
||||
audio = payload[key]
|
||||
audio = torch.cat(audio) if isinstance(audio, list) else audio
|
||||
assert audio.tolist() == [10, 11]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("audio", [None, [], torch.empty(0)])
|
||||
@pytest.mark.parametrize("codec_overtook", [False, True])
|
||||
async def test_invalid_first_audio_reports_error_and_releases_request(audio, codec_overtook):
|
||||
obj = _orchestrator()
|
||||
obj.request_states["healthy"] = OrchestratorRequestState(request_id="healthy", final_stage_id=1)
|
||||
if codec_overtook:
|
||||
await obj._route_upstream_first_audio(1, 2, _raw(required=True, terminal=True))
|
||||
assert obj.request_states["r"].pending_first_audio_outputs
|
||||
first = _raw(first=True)
|
||||
first.outputs[0].multimodal_output["model_outputs"] = audio
|
||||
await obj._route_upstream_first_audio(0, 0, first)
|
||||
|
||||
error = obj.output_async_queue.get_nowait()
|
||||
assert isinstance(error, ErrorMessage)
|
||||
assert error.request_id == "r" and "first-audio" in error.error
|
||||
assert not first.outputs and set(obj.request_states) == {"healthy"}
|
||||
obj._abort_request_ids.assert_awaited_once_with(["r"])
|
||||
obj._release_request_bindings.assert_called_once_with(["r"])
|
||||
await obj._route_upstream_first_audio(0, 0, _raw(first=True))
|
||||
assert obj.output_async_queue.empty()
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Cache isolation and graph replay for the opt-in TTS predictor."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.common.qwen3_code_predictor import CodePredictorBaseModel
|
||||
from vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts import Qwen3TTSTalkerCodePredictorConfig
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cuda]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available() or torch.version.hip is not None, reason="requires CUDA")
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_cached_frames_match_causal_prefill_after_graph_replay(dtype):
|
||||
from vllm_omni.model_executor.models.qwen3_tts.cached_code_predictor import FrameLocalKVCache
|
||||
|
||||
config = Qwen3TTSTalkerCodePredictorConfig(
|
||||
vocab_size=64,
|
||||
hidden_size=64,
|
||||
intermediate_size=128,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=4,
|
||||
num_key_value_heads=2,
|
||||
head_dim=16,
|
||||
num_code_groups=4,
|
||||
)
|
||||
torch.manual_seed(171)
|
||||
model = CodePredictorBaseModel(config).to(device="cuda", dtype=dtype).eval()
|
||||
inputs = torch.randn(3, 5, 64, device="cuda", dtype=dtype)
|
||||
positions = torch.arange(5, device="cuda").expand(3, -1)
|
||||
cache = FrameLocalKVCache(model, max_batch=3)
|
||||
with torch.inference_mode():
|
||||
for batch in (1, 3):
|
||||
for _ in range(2):
|
||||
for step in (1, 2, 3):
|
||||
cache(inputs, batch, step)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = torch.cat([cache(inputs, 3, step) for step in (1, 2, 3)], dim=1)
|
||||
for _ in range(3):
|
||||
inputs.normal_()
|
||||
# No previous frame (including a smaller batch) may leak through.
|
||||
cache(inputs, 1, 1)
|
||||
cache.buffer.fill_(torch.nan)
|
||||
graph.replay()
|
||||
expected = model(inputs, positions)[:, :4]
|
||||
tolerance = 1e-5 if dtype == torch.float32 else 0.04
|
||||
torch.testing.assert_close(output, expected, atol=tolerance, rtol=tolerance)
|
||||
@@ -681,6 +681,7 @@ class TestCodePredictorGraphReplay:
|
||||
sampling_mode="per_call",
|
||||
return_proj_buf=False,
|
||||
)
|
||||
predictor._frame_cache = None
|
||||
predictor._prefix_graphs_enabled = False
|
||||
predictor._prefix_reprefill_enabled = False
|
||||
predictor._bucket_pos_ids = {1: torch.arange(4).unsqueeze(0)}
|
||||
@@ -1398,3 +1399,36 @@ class TestMTPExecutionBuckets:
|
||||
)
|
||||
wrapper2._bucket_sizes = [1, 2, 4] # stand-in for warmed-up state
|
||||
wrapper2.configure_mtp_execution_buckets([3])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
def test_tts_cache_falls_back_on_cpu(mocker, loaded_target_classes, enabled):
|
||||
cp_config, talker_config = _make_tiny_config(loaded_target_classes)
|
||||
cfg = _make_vllm_config(mocker)
|
||||
cfg.model_config.stage_connector_config = {"extra": {"code_predictor_kv_cache": enabled}}
|
||||
predictor = loaded_target_classes[2](vllm_config=cfg, config=cp_config, talker_config=talker_config)
|
||||
common = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"]
|
||||
setup = mocker.patch.object(common.CodePredictorWrapper, "_setup_compile")
|
||||
predictor._setup_compile()
|
||||
setup.assert_called_once_with()
|
||||
assert predictor._frame_cache is None
|
||||
|
||||
|
||||
def test_shared_predictor_ignores_tts_fast_path_options(mocker, loaded_target_classes):
|
||||
cp_config, _ = _make_tiny_config(loaded_target_classes)
|
||||
cfg = _make_vllm_config(mocker)
|
||||
cfg.model_config.stage_connector_config = {
|
||||
"extra": {"code_predictor_kv_cache": True, "code_predictor_fused_sampling": True}
|
||||
}
|
||||
common = sys.modules["vllm_omni.model_executor.models.common.qwen3_code_predictor"]
|
||||
predictor = common.CodePredictorWrapper(
|
||||
vllm_config=cfg,
|
||||
cp_config=cp_config,
|
||||
wrapper_config=common.CodePredictorWrapperConfig(),
|
||||
)
|
||||
assert not hasattr(predictor, "_frame_cache")
|
||||
logits = torch.randn(3, cp_config.vocab_size)
|
||||
uniforms = torch.rand_like(logits)
|
||||
expected = (logits.float() - torch.log(-torch.log(uniforms))).argmax(-1, keepdim=True)
|
||||
actual = predictor._sample_per_call(logits, 1.0, 0, None, uniforms)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Supplied-uniform samples must match the existing ATen path exactly."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cuda,
|
||||
pytest.mark.skipif(not torch.cuda.is_available() or torch.version.hip is not None, reason="requires CUDA"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch", [1, 64])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
@pytest.mark.parametrize("top_k", [-1, 1, 50, 2048])
|
||||
def test_topk_gumbel_exact_samples_and_graph(batch, dtype, top_k):
|
||||
from vllm_omni.model_executor.models.qwen3_tts.code_predictor_sampling import sample_code_topk_gumbel
|
||||
|
||||
with torch.inference_mode():
|
||||
torch.manual_seed(42)
|
||||
logits = torch.randn(batch, 2048, device="cuda", dtype=dtype) * 5
|
||||
uniforms = torch.rand(batch, 15, 2048, device="cuda")[:, 7, :]
|
||||
uniforms.clamp_(1e-6, 1 - 1e-6)
|
||||
for case in ("random", "ties", "all_masked", "part_masked", "uniform_zero", "uniform_one", "nan"):
|
||||
if case == "ties":
|
||||
logits.zero_()
|
||||
uniforms.fill_(0.5)
|
||||
elif case == "all_masked":
|
||||
logits.fill_(-torch.inf)
|
||||
elif case == "part_masked":
|
||||
logits[:, 4:9] = 1
|
||||
elif case == "uniform_zero":
|
||||
logits.normal_()
|
||||
uniforms.zero_()
|
||||
elif case == "uniform_one":
|
||||
uniforms.fill_(1)
|
||||
elif case == "nan":
|
||||
uniforms.fill_(0.5)
|
||||
logits[:, 7] = torch.nan
|
||||
scaled = logits * (1 / 0.9)
|
||||
if top_k > 0:
|
||||
threshold = scaled.topk(top_k, dim=-1).values[:, -1:]
|
||||
scaled = scaled.masked_fill(scaled < threshold, -torch.inf)
|
||||
expected = (scaled.float() - torch.log(-torch.log(uniforms))).argmax(-1, keepdim=True)
|
||||
actual = sample_code_topk_gumbel(logits, uniforms, top_k, 1 / 0.9)
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = sample_code_topk_gumbel(logits, uniforms, top_k, 1 / 0.9)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(captured, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("top_k", [0, 1, 50, 97])
|
||||
def test_topk_gumbel_accepts_strided_supplied_uniforms(top_k):
|
||||
from vllm_omni.model_executor.models.qwen3_tts.code_predictor_sampling import sample_code_topk_gumbel
|
||||
|
||||
torch.manual_seed(81)
|
||||
logits = torch.randn(3, 194, device="cuda", dtype=torch.bfloat16)[:, ::2]
|
||||
uniforms = torch.rand(3, 15, 194, device="cuda")[:, 3, ::2].clamp_(1e-6, 1 - 1e-6)
|
||||
scaled = logits * (1 / 0.9)
|
||||
if top_k > 0:
|
||||
scaled = scaled.masked_fill(scaled < scaled.topk(top_k).values[:, -1:], -torch.inf)
|
||||
expected = (scaled.float() - torch.log(-torch.log(uniforms))).argmax(-1, keepdim=True)
|
||||
actual = sample_code_topk_gumbel(logits, uniforms, top_k, 1 / 0.9)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Prefix graphs must follow per-request first-audio delivery, not a capture hint."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.model_executor.models.qwen3_tts.test_qwen3_tts_incremental_decode import _decoder_stub
|
||||
from vllm_omni.model_executor.models.qwen3_tts.segmented_graph_wrapper import CUDAGraphDecoderWrapper
|
||||
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
|
||||
Qwen3TTSTokenizerV2Decoder,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model]
|
||||
DEVICE = torch.device("cuda:0")
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("skip_flags", [(False, False), (True, True), (False, True), (True, False)])
|
||||
@pytest.mark.parametrize("initial_frames", [1, 3])
|
||||
@pytest.mark.parametrize("state_only_available", [False, True])
|
||||
def test_xvec_first_chunk_replays_prefix_graph(monkeypatch, skip_flags, initial_frames, state_only_available):
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
|
||||
wrapper = CUDAGraphDecoderWrapper.__new__(CUDAGraphDecoderWrapper)
|
||||
wrapper.prefix_length = 72
|
||||
wrapper.initial_chunk_frames = initial_frames
|
||||
wrapper.codec_chunk_frames = 25
|
||||
wrapper.capture_batch_sizes = [1, 2]
|
||||
wrapper._icl_previous_frames_by_target = {26: 1}
|
||||
wrapper._xvec_previous_frames_by_target = {initial_frames + 25: initial_frames}
|
||||
replayed = []
|
||||
wrapper.xvec_prefix_states = {
|
||||
2: {
|
||||
"graph": SimpleNamespace(replay=lambda: replayed.append("full")),
|
||||
"input": {"codes": torch.zeros(2, 2, initial_frames)},
|
||||
"output": torch.arange(4 * initial_frames, dtype=torch.float32).view(2, 1, 2 * initial_frames),
|
||||
"cache": {
|
||||
"ref_hidden": torch.zeros(2, 2, 0),
|
||||
"ref_conv": torch.zeros(2, 0, 3),
|
||||
"prefix_hidden": torch.zeros(2, 0, 3),
|
||||
"ref_upsample": torch.zeros(2, 3, 0),
|
||||
"ref_wav": torch.zeros(2, 1, 0),
|
||||
"suffix_quantized": torch.ones(2, 2, initial_frames),
|
||||
"suffix_conv": torch.ones(2, initial_frames, 3),
|
||||
"past_key_values": SimpleNamespace(layers=[]),
|
||||
},
|
||||
}
|
||||
}
|
||||
wrapper.xvec_prefix_state_only_states = {}
|
||||
if state_only_available and initial_frames == 1:
|
||||
wrapper.xvec_prefix_state_only_states[2] = {
|
||||
**wrapper.xvec_prefix_states[2],
|
||||
"graph": SimpleNamespace(replay=lambda: replayed.append("state")),
|
||||
"state_only": True,
|
||||
}
|
||||
wrapper._record_graph_hit = lambda *_args: None
|
||||
wrapper._record_graph_fallback = lambda *_args: None
|
||||
wrapper._ensure_suffix_buffers = lambda cache: None
|
||||
wrapper.decoder = _decoder_stub(
|
||||
capture_first_audio_state_only=True,
|
||||
total_upsample=2,
|
||||
_is_suffix_cache_rolling=lambda previous, cached: False,
|
||||
_decode_xvec_first_chunk=lambda *_args: pytest.fail("unexpected eager fallback"),
|
||||
_slice_dynamic_cache=Qwen3TTSTokenizerV2Decoder._slice_dynamic_cache,
|
||||
)
|
||||
|
||||
caches = [{"prefix_frames": 0, "skip_first_audio": skip} for skip in skip_flags]
|
||||
outputs = wrapper._batched_request_decode(
|
||||
[torch.full((1, 2, initial_frames), 3), torch.full((1, 2, initial_frames), 4)],
|
||||
caches,
|
||||
)
|
||||
|
||||
assert replayed == ["state" if all(skip_flags) and state_only_available and initial_frames == 1 else "full"]
|
||||
torch.testing.assert_close(wrapper.xvec_prefix_states[2]["input"]["codes"][0], torch.full((2, initial_frames), 3.0))
|
||||
torch.testing.assert_close(wrapper.xvec_prefix_states[2]["input"]["codes"][1], torch.full((2, initial_frames), 4.0))
|
||||
assert [cache["decoder_prefix_frames"] for cache in caches] == [0, 0]
|
||||
assert [cache["suffix_frames"] for cache in caches] == [initial_frames, initial_frames]
|
||||
assert len(outputs) == 2
|
||||
for row, skip in enumerate(skip_flags):
|
||||
expected = wrapper.xvec_prefix_states[2]["output"][row : row + 1, :, 2 if skip else 0 :]
|
||||
torch.testing.assert_close(outputs[row], expected)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("initial_frames", [1, 3])
|
||||
@pytest.mark.parametrize("time_major", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_xvec_graphs_follow_request_first_audio_flags(initial_frames, time_major):
|
||||
from tests.model_executor.models.qwen3_tts.test_time_major_decoder import _make_decoder
|
||||
from vllm_omni.model_executor.models.qwen3_tts.segmented_graph_wrapper import (
|
||||
CUDAGraphDecoderWrapper as SegmentedWrapper,
|
||||
)
|
||||
|
||||
decoder = _make_decoder().to(DEVICE)
|
||||
decoder.precompute_snake_caches()
|
||||
if time_major:
|
||||
decoder.enable_time_major_conv()
|
||||
decoder.capture_first_audio_state_only = True
|
||||
wrapper = SegmentedWrapper(
|
||||
decoder,
|
||||
capture_modes=("xvec",),
|
||||
capture_batch_sizes=[1, 2],
|
||||
num_quantizers=2,
|
||||
initial_chunk_frames=initial_frames,
|
||||
)
|
||||
# This regression covers prefix graphs. Avoid compiling unrelated suffix
|
||||
# shapes, especially the time-major kernels for long continuation chunks.
|
||||
wrapper._xvec_previous_frames_by_target = {}
|
||||
wrapper.warmup(DEVICE)
|
||||
assert set(wrapper.xvec_prefix_states) == {1, 2}
|
||||
assert set(wrapper.xvec_prefix_state_only_states) == ({1, 2} if initial_frames == 1 else set())
|
||||
|
||||
for skip_flags in ((False, False), (True, True), (False, True), (True, False)):
|
||||
codes = [torch.randint(0, 32, (1, 2, initial_frames), device=DEVICE) for _ in skip_flags]
|
||||
caches = [{"prefix_frames": 0, "skip_first_audio": skip} for skip in skip_flags]
|
||||
expected_caches = [{"prefix_frames": 0, "skip_first_audio": skip} for skip in skip_flags]
|
||||
expected = [decoder._decode_stream_first_chunk(code, cache) for code, cache in zip(codes, expected_caches)]
|
||||
actual = wrapper._decode_xvec_prefix_batch(codes, caches)
|
||||
assert actual is not None # Eligible batches must not silently become per-request eager calls.
|
||||
for got, want, cache, expected_cache in zip(actual, expected, caches, expected_caches):
|
||||
torch.testing.assert_close(got, want, atol=1e-4, rtol=1e-4)
|
||||
for key in ("ref_hidden", "ref_conv", "prefix_hidden", "suffix_quantized", "suffix_conv"):
|
||||
torch.testing.assert_close(cache[key], expected_cache[key], atol=1e-5, rtol=1e-5)
|
||||
@@ -752,3 +752,37 @@ def test_load_weights_uses_model_dtype_before_precomputing_caches(dtype):
|
||||
assert _load_weights_noop(model) == {"decoder.fake_weight"}
|
||||
assert model.decoder.weight.dtype is dtype
|
||||
assert cache_dtypes == [dtype]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capture_fails", [False, True])
|
||||
def test_decode_autotune_restores_process_flags(mocker, monkeypatch, capture_fails):
|
||||
monkeypatch.setattr(torch.backends.cudnn, "benchmark", False)
|
||||
monkeypatch.setattr(torch.backends.cudnn, "benchmark_limit", 5)
|
||||
original = (torch.backends.cudnn.enabled, torch.backends.cudnn.deterministic, torch.backends.cudnn.allow_tf32)
|
||||
model = _make_model(
|
||||
async_chunk=True,
|
||||
device=torch.device("cuda"),
|
||||
stage_connector_config={"extra": {"decode_cudnn_benchmark": True}},
|
||||
)
|
||||
|
||||
observed = []
|
||||
|
||||
def capture(**kwargs):
|
||||
observed.append(
|
||||
(
|
||||
torch.backends.cudnn.benchmark,
|
||||
torch.backends.cudnn.benchmark_limit,
|
||||
torch.backends.cudnn.enabled,
|
||||
torch.backends.cudnn.deterministic,
|
||||
torch.backends.cudnn.allow_tf32,
|
||||
)
|
||||
)
|
||||
if capture_fails:
|
||||
raise RuntimeError("capture failed")
|
||||
|
||||
capture_mock = mocker.patch.object(model, "_maybe_enable_decoder_cudagraph", side_effect=capture)
|
||||
_load_weights_noop(model)
|
||||
capture_mock.assert_called_once()
|
||||
assert observed == [(True, 10, *original)]
|
||||
assert torch.backends.cudnn.benchmark is False
|
||||
assert torch.backends.cudnn.benchmark_limit == 5
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Causal/GQA correctness and graph replay for the frame-local KV kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cuda,
|
||||
pytest.mark.skipif(not torch.cuda.is_available() or torch.version.hip is not None, reason="requires CUDA"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch", [1, 3, 64])
|
||||
@pytest.mark.parametrize("heads,kv_heads,dim", [(16, 8, 128), (8, 8, 64)])
|
||||
@pytest.mark.parametrize("positions", [[0, 1], [2], [8], [15]])
|
||||
def test_short_attention_causal_gqa_and_graph(batch, heads, kv_heads, dim, positions):
|
||||
from vllm_omni.model_executor.models.qwen3_tts.short_kv_attention import short_kv_attention
|
||||
|
||||
with torch.inference_mode():
|
||||
torch.manual_seed(42)
|
||||
pos = torch.tensor(positions, device="cuda")
|
||||
q = torch.randn(batch, len(positions), heads, dim, device="cuda", dtype=torch.bfloat16).transpose(1, 2)
|
||||
cache = torch.randn(2, batch + 3, kv_heads, 17, dim, device="cuda", dtype=torch.bfloat16)
|
||||
k, v = cache[0, :batch], cache[1, :batch]
|
||||
mask = torch.arange(17, device="cuda")[None, :] <= pos[:, None]
|
||||
kr = k.float().repeat_interleave(heads // kv_heads, dim=1)
|
||||
vr = v.float().repeat_interleave(heads // kv_heads, dim=1)
|
||||
scores = (q.float() @ kr.transpose(-1, -2)) * dim**-0.5
|
||||
expected = scores.masked_fill(~mask, -torch.inf).softmax(-1) @ vr
|
||||
actual = short_kv_attention(q, k, v, pos, dim**-0.5)
|
||||
torch.testing.assert_close(actual.float(), expected, atol=0.016, rtol=0.008)
|
||||
|
||||
# Unwritten cache slots must not affect a new frame, even if poisoned.
|
||||
k[:, :, max(positions) + 1 :] = torch.nan
|
||||
v[:, :, max(positions) + 1 :] = torch.nan
|
||||
poisoned = short_kv_attention(q, k, v, pos, dim**-0.5)
|
||||
torch.testing.assert_close(poisoned, actual, atol=0, rtol=0)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = short_kv_attention(q, k, v, pos, dim**-0.5)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(captured, actual, atol=0, rtol=0)
|
||||
@@ -0,0 +1,65 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Code2Wav drops only frames explicitly marked as delivered by the Talker."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
|
||||
Qwen3TTSTokenizerV2Decoder,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
UPSAMPLE = 4
|
||||
|
||||
|
||||
def _decoder(skip: bool):
|
||||
decoder = Qwen3TTSTokenizerV2Decoder.__new__(Qwen3TTSTokenizerV2Decoder)
|
||||
decoder.__dict__["capture_first_audio_state_only"] = skip
|
||||
decoder.__dict__["total_upsample"] = UPSAMPLE
|
||||
calls = []
|
||||
|
||||
def full(codes, caches):
|
||||
calls.append("full")
|
||||
frames = int(codes.shape[-1])
|
||||
return torch.arange(frames * UPSAMPLE, dtype=torch.float32).view(1, 1, -1)
|
||||
|
||||
def state_only(codes, caches):
|
||||
calls.append("state")
|
||||
return torch.zeros(1)
|
||||
|
||||
decoder.__dict__["_decode_xvec_first_chunk"] = full
|
||||
decoder.__dict__["_decode_xvec_first_chunk_state_only"] = state_only
|
||||
return decoder, calls
|
||||
|
||||
|
||||
def test_one_frame_first_chunk_only_builds_state():
|
||||
decoder, calls = _decoder(skip=True)
|
||||
out = decoder._decode_stream_first_chunk(torch.zeros(1, 16, 1, dtype=torch.long), {"skip_first_audio": True})
|
||||
assert out.shape[-1] == 0 and calls == ["state"]
|
||||
|
||||
|
||||
def test_longer_first_chunk_keeps_all_but_the_delivered_frame():
|
||||
decoder, calls = _decoder(skip=True)
|
||||
out = decoder._decode_stream_first_chunk(
|
||||
torch.zeros(1, 16, 3, dtype=torch.long),
|
||||
{"skip_first_audio": True},
|
||||
)
|
||||
assert calls == ["full"]
|
||||
assert torch.equal(out, torch.arange(UPSAMPLE, 3 * UPSAMPLE, dtype=torch.float32).view(1, 1, -1))
|
||||
|
||||
|
||||
def test_without_skip_the_whole_first_chunk_is_decoded():
|
||||
decoder, calls = _decoder(skip=False)
|
||||
out = decoder._decode_stream_first_chunk(
|
||||
torch.zeros(1, 16, 3, dtype=torch.long),
|
||||
{"skip_first_audio": False},
|
||||
)
|
||||
assert calls == ["full"] and out.shape[-1] == 3 * UPSAMPLE
|
||||
|
||||
|
||||
def test_request_without_first_audio_uses_regular_decoder_even_when_graph_skip_is_enabled():
|
||||
decoder, calls = _decoder(skip=True)
|
||||
out = decoder._decode_stream_first_chunk(torch.zeros(1, 16, 1, dtype=torch.long), {})
|
||||
assert calls == ["full"] and out.shape[-1] == UPSAMPLE
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz import time_major_conv as tmc
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _snake_nct(x, exp_alpha, inv_beta):
|
||||
return x + inv_beta[None, :, None] * torch.sin(x * exp_alpha[None, :, None]).square()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("kernel_size", "dilation"), [(1, 1), (7, 1), (7, 3), (3, 9)])
|
||||
@pytest.mark.parametrize("with_bias_and_snake", [False, True])
|
||||
def test_im2col_gemm_matches_causal_conv1d(kernel_size, dilation, with_bias_and_snake):
|
||||
torch.manual_seed(0)
|
||||
bsz, t, c_in, c_out = 2, 11, 6, 5
|
||||
conv = torch.nn.Conv1d(c_in, c_out, kernel_size, dilation=dilation)
|
||||
x = torch.randn(bsz, c_in, t)
|
||||
in_bias = torch.randn(c_in) if with_bias_and_snake else None
|
||||
snake = (torch.rand(c_in) + 0.5, torch.rand(c_in) + 0.5) if with_bias_and_snake else None
|
||||
|
||||
ref_in = x if in_bias is None else x + in_bias[None, :, None]
|
||||
if snake is not None:
|
||||
ref_in = _snake_nct(ref_in, *snake)
|
||||
expected = conv(F.pad(ref_in, ((kernel_size - 1) * dilation, 0)))
|
||||
|
||||
cols = tmc.snake_im2col(x.transpose(1, 2).contiguous(), kernel_size, dilation, in_bias, snake)
|
||||
actual = torch.addmm(conv.bias, cols, tmc.conv_gemm_weight(conv.weight)).view(bsz, t, c_out)
|
||||
torch.testing.assert_close(actual.transpose(1, 2), expected, rtol=1e-5, atol=1e-5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("stride", "kernel_size"), [(2, 2), (3, 6), (8, 16)])
|
||||
def test_transposed_overlap_add_matches_trimmed_conv_transpose(stride, kernel_size):
|
||||
torch.manual_seed(0)
|
||||
bsz, t, c_in, c_out = 2, 5, 4, 3
|
||||
conv = torch.nn.ConvTranspose1d(c_in, c_out, kernel_size, stride=stride)
|
||||
x = torch.randn(bsz, c_in, t)
|
||||
expected = conv(x)[..., : t * stride] # causal: drop the kernel - stride right tail
|
||||
|
||||
z = (x.transpose(1, 2).reshape(bsz * t, c_in) @ tmc.transposed_gemm_weight(conv.weight)).view(
|
||||
bsz, t, kernel_size, c_out
|
||||
)
|
||||
actual = tmc.transposed_overlap_add(z, stride, conv.bias)
|
||||
torch.testing.assert_close(actual.transpose(1, 2), expected, rtol=1e-5, atol=1e-5)
|
||||
|
||||
|
||||
def test_transposed_overlap_add_rejects_other_kernels():
|
||||
with pytest.raises(ValueError, match="kernel stride or 2"):
|
||||
tmc.transposed_overlap_add(torch.zeros(1, 2, 5, 3), 2, torch.zeros(3))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("kernel_size", "dilation"), [(1, 1), (7, 3)])
|
||||
@pytest.mark.parametrize("with_residual", [False, True])
|
||||
def test_causal_conv_matches_conv1d(kernel_size, dilation, with_residual):
|
||||
torch.manual_seed(0)
|
||||
bsz, t, c_in, c_out = 2, 13, 6, 5
|
||||
conv = torch.nn.Conv1d(c_in, c_out, kernel_size, dilation=dilation)
|
||||
x = torch.randn(bsz, c_in, t)
|
||||
residual = torch.randn(bsz, t, c_out) if with_residual else None
|
||||
expected = conv(F.pad(x, ((kernel_size - 1) * dilation, 0))).transpose(1, 2)
|
||||
if residual is not None:
|
||||
expected = expected + residual
|
||||
actual = tmc.causal_conv(
|
||||
x.transpose(1, 2).contiguous(), tmc.conv_gemm_weight(conv.weight), conv.bias, kernel_size, dilation, residual
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
|
||||
|
||||
|
||||
def _gpu_tolerances(dtype):
|
||||
# bf16 results may differ by one ulp (sin rounding, fp32 vs bf16 accumulation order).
|
||||
return {"rtol": 1.6e-2, "atol": 1e-2} if dtype == torch.bfloat16 else {"rtol": 1e-4, "atol": 1e-4}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available() or not tmc.HAS_TRITON, reason="needs CUDA and Triton")
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_triton_kernels_match_reference(dtype):
|
||||
torch.manual_seed(0)
|
||||
tol = _gpu_tolerances(dtype)
|
||||
x = torch.randn(3, 37, 96, device="cuda", dtype=dtype)
|
||||
bias = torch.randn(96, device="cuda", dtype=dtype)
|
||||
snake = (torch.rand(96, device="cuda", dtype=dtype) + 0.5, torch.rand(96, device="cuda", dtype=dtype) + 0.5)
|
||||
for kernel_size, dilation in ((1, 1), (7, 9)):
|
||||
torch.testing.assert_close(
|
||||
tmc.snake_im2col(x, kernel_size, dilation, bias, snake),
|
||||
tmc.snake_im2col_reference(x, kernel_size, dilation, bias, snake),
|
||||
**tol,
|
||||
)
|
||||
z = torch.randn(3, 37, 10, 48, device="cuda", dtype=dtype)
|
||||
torch.testing.assert_close(
|
||||
tmc.transposed_overlap_add(z, 5, bias[:48]), tmc.transposed_overlap_add_reference(z, 5, bias[:48]), **tol
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available() or not tmc.HAS_TRITON, reason="needs CUDA and Triton")
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize(("c_in", "c_out", "kernel_size", "dilation"), [(96, 96, 7, 9), (192, 96, 1, 1), (96, 1, 7, 1)])
|
||||
def test_triton_causal_conv_matches_reference(dtype, c_in, c_out, kernel_size, dilation):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(3, 101, c_in, device="cuda", dtype=dtype)
|
||||
weight = torch.randn(kernel_size * c_in, c_out, device="cuda", dtype=dtype) / (kernel_size * c_in) ** 0.5
|
||||
bias = torch.randn(c_out, device="cuda", dtype=dtype)
|
||||
residual = torch.randn(3, 101, c_out, device="cuda", dtype=dtype)
|
||||
expected = tmc.causal_conv_reference(
|
||||
x.float(), weight.float(), bias.float(), kernel_size, dilation, residual.float()
|
||||
)
|
||||
actual = tmc.causal_conv(x, weight, bias, kernel_size, dilation, residual)
|
||||
torch.testing.assert_close(actual.float(), expected, **_gpu_tolerances(dtype))
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import (
|
||||
Qwen3TTSTokenizerV2DecoderConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
|
||||
Qwen3TTSTokenizerV2Decoder,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_decoder() -> Qwen3TTSTokenizerV2Decoder:
|
||||
config = Qwen3TTSTokenizerV2DecoderConfig(
|
||||
codebook_size=32,
|
||||
hidden_size=16,
|
||||
latent_dim=16,
|
||||
codebook_dim=16,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
intermediate_size=32,
|
||||
num_hidden_layers=1,
|
||||
num_quantizers=2,
|
||||
decoder_dim=32,
|
||||
upsample_rates=(8, 5, 4, 3),
|
||||
upsampling_ratios=(2, 2),
|
||||
sliding_window=72,
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
decoder = Qwen3TTSTokenizerV2Decoder(config).eval()
|
||||
with torch.no_grad():
|
||||
for name, param in decoder.named_parameters():
|
||||
# Non-trivial SnakeBeta and ConvNeXt scales so every term matters.
|
||||
if name.endswith(("alpha", "beta")):
|
||||
param.uniform_(-0.5, 0.5)
|
||||
elif name.endswith("gamma"):
|
||||
param.uniform_(0.5, 1.0)
|
||||
decoder.precompute_snake_caches()
|
||||
return decoder
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_time_major_conv_decode_matches_nct_stack() -> None:
|
||||
decoder = _make_decoder()
|
||||
hidden = torch.randn(2, 5, decoder.config.latent_dim)
|
||||
expected = decoder._conv_decode(hidden)
|
||||
|
||||
decoder.enable_time_major_conv()
|
||||
actual = decoder._conv_decode(hidden)
|
||||
|
||||
assert actual.shape == expected.shape == (2, 1, 5 * int(decoder.total_upsample))
|
||||
scale = expected.abs().max()
|
||||
torch.testing.assert_close(actual / scale, expected / scale, rtol=0, atol=1e-5)
|
||||
# The stack holds weight copies outside the module tree.
|
||||
assert not any("time_major" in key for key in decoder.state_dict())
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_time_major_conv_decode_takes_sliced_hidden() -> None:
|
||||
decoder = _make_decoder()
|
||||
hidden = torch.randn(1, 9, decoder.config.latent_dim)
|
||||
expected = decoder._conv_decode(hidden[:, -4:, :])
|
||||
decoder.enable_time_major_conv()
|
||||
actual = decoder._conv_decode(hidden[:, -4:, :]) # non-contiguous, like the suffix decode's slice
|
||||
scale = expected.abs().max()
|
||||
torch.testing.assert_close(actual / scale, expected / scale, rtol=0, atol=1e-5)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_first_frame_single_key_attention_matches_codec_without_graphs():
|
||||
from vllm_omni.model_executor.models.qwen3_tts.first_frame_decoder import Qwen3TTSFirstFrameDecoder
|
||||
|
||||
decoder = _make_decoder()
|
||||
codes = torch.randint(0, 32, (3, 2, 1))
|
||||
expected = decoder._decode_xvec_first_chunk(codes, {})[:, 0, :]
|
||||
first = Qwen3TTSFirstFrameDecoder.__new__(Qwen3TTSFirstFrameDecoder)
|
||||
torch.nn.Module.__init__(first)
|
||||
first.decoder = decoder
|
||||
first._graphs = {}
|
||||
actual = first.decode(codes[:, :, 0])
|
||||
torch.testing.assert_close(actual, expected, rtol=1e-4, atol=1e-5)
|
||||
@@ -0,0 +1,237 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import queue
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.engine import OmniEngineCoreOutputs
|
||||
from vllm_omni.worker_v2.first_audio_sender import engine_output_queue_sink
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _scheduler(**client_by_request):
|
||||
return SimpleNamespace(
|
||||
requests={request_id: SimpleNamespace(client_index=client) for request_id, client in client_by_request.items()}
|
||||
)
|
||||
|
||||
|
||||
def test_sink_groups_outputs_by_client_and_carries_audio():
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(a=0, b=1, c=0))
|
||||
sample_rate = torch.tensor(24000, dtype=torch.int32)
|
||||
rows = [torch.full((4,), float(i)) for i in range(3)]
|
||||
|
||||
sink.prepare(["a", "b", "c"])(["a", "b", "c"], rows, sample_rate)
|
||||
|
||||
delivered = {}
|
||||
while not output_queue.empty():
|
||||
client_index, outputs = output_queue.get_nowait()
|
||||
assert isinstance(outputs, OmniEngineCoreOutputs)
|
||||
delivered[client_index] = outputs.outputs
|
||||
assert sorted(delivered) == [0, 1]
|
||||
assert [o.request_id for o in delivered[0]] == ["a", "c"]
|
||||
assert [o.request_id for o in delivered[1]] == ["b"]
|
||||
for output in delivered[0] + delivered[1]:
|
||||
# First audio carries no tokens and does not finish the request.
|
||||
assert output.new_token_ids == []
|
||||
assert output.finish_reason is None
|
||||
assert output.multimodal_output["sr"] is sample_rate
|
||||
assert torch.equal(delivered[0][1].multimodal_output["model_outputs"], rows[2])
|
||||
|
||||
|
||||
def test_sink_skips_requests_that_already_left_the_scheduler():
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(kept=0))
|
||||
|
||||
sink.prepare(["gone", "kept"])(["gone", "kept"], [torch.zeros(2), torch.ones(2)], torch.tensor(24000))
|
||||
sink.prepare(["gone"])(["gone"], [torch.zeros(2)], torch.tensor(24000))
|
||||
|
||||
client_index, outputs = output_queue.get_nowait()
|
||||
assert client_index == 0
|
||||
assert [o.request_id for o in outputs.outputs] == ["kept"]
|
||||
assert output_queue.empty()
|
||||
|
||||
|
||||
class _DoneEvent:
|
||||
def record(self):
|
||||
pass
|
||||
|
||||
def synchronize(self):
|
||||
pass
|
||||
|
||||
|
||||
def _run_sender(items):
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(**{rid: 0 for item in items for rid in item[2]}))
|
||||
sender = FirstAudioSender(sink)
|
||||
for item in items:
|
||||
sender._queue.put((*item, sink.prepare(item[2])))
|
||||
sender.close()
|
||||
delivered = []
|
||||
while not output_queue.empty():
|
||||
_, outputs = output_queue.get_nowait()
|
||||
delivered.append(
|
||||
([o.request_id for o in outputs.outputs], [o.multimodal_output["model_outputs"] for o in outputs.outputs])
|
||||
)
|
||||
return delivered
|
||||
|
||||
|
||||
def test_sender_drops_rows_whose_frame_is_not_audio():
|
||||
pcm = torch.arange(6, dtype=torch.float32).reshape(3, 2)
|
||||
sample_rate = torch.tensor(24000, dtype=torch.int32)
|
||||
valid = torch.tensor([True, False, True])
|
||||
|
||||
delivered = _run_sender(
|
||||
[
|
||||
(_DoneEvent(), pcm, ["a", "b", "c"], sample_rate, valid),
|
||||
# Every row invalid: nothing reaches the sink.
|
||||
(_DoneEvent(), pcm[:1], ["d"], sample_rate, torch.tensor([False])),
|
||||
# No validity: every row is delivered (previous behaviour).
|
||||
(_DoneEvent(), pcm[:1], ["e"], sample_rate, None),
|
||||
]
|
||||
)
|
||||
|
||||
assert [ids for ids, _rows in delivered] == [["a", "c"], ["e"]]
|
||||
assert torch.equal(delivered[0][1][1], pcm[2])
|
||||
|
||||
|
||||
def test_prepared_delivery_survives_scheduler_completion():
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
scheduler = _scheduler(r=3)
|
||||
sink = engine_output_queue_sink(output_queue, scheduler)
|
||||
deliver = sink.prepare(["r"])
|
||||
scheduler.requests.clear()
|
||||
deliver(["r"], [torch.ones(2)], torch.tensor(24000))
|
||||
client, outputs = output_queue.get_nowait()
|
||||
assert client == 3
|
||||
assert outputs.outputs[0].multimodal_output["_omni_first_audio"]
|
||||
|
||||
|
||||
def test_delivery_failure_emits_request_error():
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
delivery = engine_output_queue_sink(output_queue, _scheduler(r=0)).prepare(["r"])
|
||||
delivery.fail(["r"])
|
||||
_, outputs = output_queue.get_nowait()
|
||||
assert outputs.outputs[0].finish_reason == FinishReason.ERROR
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_index", [0, 3])
|
||||
def test_submit_reports_only_requests_with_a_prepared_route(monkeypatch, client_index):
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
# Exercise the routing contract without allocating CUDA or pinned memory.
|
||||
original_empty = torch.empty
|
||||
monkeypatch.setattr(torch, "empty", lambda *args, pin_memory=False, **kwargs: original_empty(*args, **kwargs))
|
||||
monkeypatch.setattr(torch.cuda, "Event", _DoneEvent)
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sender = FirstAudioSender(engine_output_queue_sink(output_queue, _scheduler(kept=client_index)))
|
||||
pcm = torch.arange(6, dtype=torch.float32).reshape(3, 2)
|
||||
try:
|
||||
accepted = sender.submit(["gone", "kept", "also-gone"], pcm, torch.tensor(24000))
|
||||
finally:
|
||||
sender.close()
|
||||
|
||||
assert accepted == ["kept"]
|
||||
client, outputs = output_queue.get_nowait()
|
||||
assert client == client_index
|
||||
assert [output.request_id for output in outputs.outputs] == ["kept"]
|
||||
assert torch.equal(outputs.outputs[0].multimodal_output["model_outputs"], pcm[1])
|
||||
assert output_queue.empty()
|
||||
|
||||
|
||||
def test_submit_without_any_route_leaves_audio_to_the_codec(monkeypatch):
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
def unexpected_copy(*args, **kwargs):
|
||||
pytest.fail("No delivery is possible; do not start a device copy")
|
||||
|
||||
monkeypatch.setattr(torch, "empty", unexpected_copy)
|
||||
monkeypatch.setattr(torch.cuda, "Event", unexpected_copy)
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sender = FirstAudioSender(engine_output_queue_sink(output_queue, _scheduler()))
|
||||
try:
|
||||
accepted = sender.submit(["gone"], torch.ones(1, 2), torch.tensor(24000))
|
||||
finally:
|
||||
sender.close()
|
||||
|
||||
assert accepted == []
|
||||
assert output_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sink", [lambda ids, rows, sr: None, SimpleNamespace(prepare=None)])
|
||||
def test_sender_rejects_sink_without_route_preparation(sink):
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
with pytest.raises(TypeError, match="prepare"):
|
||||
FirstAudioSender(sink)
|
||||
|
||||
|
||||
def test_sender_failure_only_aborts_valid_undelivered_rows():
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
class FailSecondClientOnce(queue.Queue):
|
||||
def put_nowait(self, item):
|
||||
client, outputs = item
|
||||
if client == 1 and outputs.outputs[0].finish_reason is None:
|
||||
raise RuntimeError("client enqueue failed")
|
||||
super().put_nowait(item)
|
||||
|
||||
output_queue = FailSecondClientOnce()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(eos=1, sent=0, failed=1, pending=2, next=3))
|
||||
sender = FirstAudioSender(sink)
|
||||
ids = ["eos", "sent", "failed", "pending", "gone"]
|
||||
sender._queue.put(
|
||||
(
|
||||
_DoneEvent(),
|
||||
torch.ones(5, 2),
|
||||
ids,
|
||||
torch.tensor(24000),
|
||||
torch.tensor([False, True, True, True, True]),
|
||||
sink.prepare(ids),
|
||||
)
|
||||
)
|
||||
sender._queue.put((_DoneEvent(), torch.ones(1, 2), ["next"], torch.tensor(24000), None, sink.prepare(["next"])))
|
||||
sender.close()
|
||||
|
||||
actual: list[tuple[int, str, FinishReason | None]] = []
|
||||
while not output_queue.empty():
|
||||
client, outputs = output_queue.get_nowait()
|
||||
actual.extend((client, o.request_id, o.finish_reason) for o in outputs.outputs)
|
||||
assert actual == [
|
||||
(0, "sent", None),
|
||||
(1, "failed", FinishReason.ERROR),
|
||||
(2, "pending", FinishReason.ERROR),
|
||||
(3, "next", None),
|
||||
]
|
||||
|
||||
|
||||
def test_sender_copy_failure_reports_all_prepared_routes():
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
class FailedEvent:
|
||||
def synchronize(self):
|
||||
raise RuntimeError("copy failed before validity became readable")
|
||||
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(r=3))
|
||||
sender = FirstAudioSender(sink)
|
||||
sender._queue.put(
|
||||
(FailedEvent(), torch.ones(2, 2), ["r", "gone"], torch.tensor(24000), None, sink.prepare(["r", "gone"]))
|
||||
)
|
||||
sender.close()
|
||||
client, outputs = output_queue.get_nowait()
|
||||
assert client == 3
|
||||
assert [(o.request_id, o.finish_reason) for o in outputs.outputs] == [("r", FinishReason.ERROR)]
|
||||
assert output_queue.empty()
|
||||
@@ -391,3 +391,204 @@ def test_mtp_requires_model_declared_output_key(output_key):
|
||||
model = SimpleNamespace(mtp=lambda: None, mtp_output_key=output_key)
|
||||
with pytest.raises(TypeError, match="must declare mtp_output_key"):
|
||||
state._init_mtp_runner(model)
|
||||
|
||||
|
||||
_EAGER_DIM = 3
|
||||
_CODEBOOK = 2048
|
||||
_EOS = 2150
|
||||
|
||||
|
||||
class _EagerBatch(_DummyInputBatch):
|
||||
def __init__(self, spans, indices=None):
|
||||
indices = list(range(len(spans))) if indices is None else indices
|
||||
super().__init__(indices)
|
||||
starts = [0]
|
||||
for n_tok in spans:
|
||||
starts.append(starts[-1] + n_tok)
|
||||
self.num_scheduled_tokens = list(spans)
|
||||
self.query_start_loc_np = starts[:-1]
|
||||
self.query_start_loc = torch.tensor(starts, dtype=torch.int32)
|
||||
|
||||
|
||||
def _make_eager_state(max_num_reqs=4):
|
||||
state = _make_state(max_num_reqs=max_num_reqs, has_preprocess=True)
|
||||
_init_static(state, max_num_reqs, dim=_EAGER_DIM)
|
||||
from vllm_omni.worker_v2.model_states.eager_mtp import EagerMTPState
|
||||
|
||||
state._eager_state = EagerMTPState(state)
|
||||
state._first_audio_requests = set()
|
||||
state._eager_mtp = True
|
||||
state._eager_fastpath = False
|
||||
state._eager_rows = None
|
||||
state._eager_ready = {}
|
||||
state._eager_settled = {}
|
||||
state._eager_embeds = torch.zeros((max_num_reqs, _EAGER_DIM))
|
||||
state._decode_preprocess = None
|
||||
state._mtp_sample_uniforms = None
|
||||
state.vllm_config.cache_config = SimpleNamespace(enable_prefix_caching=False)
|
||||
model = state.model
|
||||
model.first_frame_decoder = None
|
||||
model.mtp_frame_valid = lambda layer0: (layer0 >= 0) & (layer0 < _CODEBOOK)
|
||||
model.embed_input_ids = lambda ids: ids.float().reshape(-1, 1, 1).expand(-1, 1, _EAGER_DIM)
|
||||
model.mtp_calls = []
|
||||
|
||||
def mtp(input_ids, input_embeds, last_hidden, text_step, **kwargs):
|
||||
model.mtp_calls.append((input_ids.clone(), last_hidden.clone(), text_step.clone()))
|
||||
codes = torch.stack([input_ids, input_ids + 1, input_ids + 2], dim=1)
|
||||
# Frame embedding sum plus the text step, like the Qwen3 talkers.
|
||||
return input_embeds + 100 + text_step, codes
|
||||
|
||||
model.mtp = mtp
|
||||
return state
|
||||
|
||||
|
||||
def _eager_outputs(num_tokens):
|
||||
return {
|
||||
"codes": {"audio": torch.zeros((num_tokens, 3), dtype=torch.long)},
|
||||
"meta": {"codec_frame_valid": torch.zeros((num_tokens,), dtype=torch.int8)},
|
||||
}
|
||||
|
||||
|
||||
def test_eager_mtp_publishes_frames_in_the_step_that_sampled_cb0():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "prefill", "decode", "ended")
|
||||
# prefill: final 3-token chunk; decode: input CB0 42; ended: input CB0 was
|
||||
# EOS (async scheduling ran the finished request once more).
|
||||
batch = _EagerBatch([3, 1, 1])
|
||||
input_ids = torch.tensor([0, 0, 0, 42, _EOS])
|
||||
state._eager_rows = (batch, [(0, 0, "prefill", True), (1, 1, "decode", False), (2, 2, "ended", False)], input_ids)
|
||||
text_hidden = torch.arange(5 * _EAGER_DIM, dtype=torch.float32).reshape(5, _EAGER_DIM)
|
||||
sampled = torch.tensor([[7], [_EOS], [9]])
|
||||
outputs = _eager_outputs(5)
|
||||
|
||||
state.run_eager_mtp(batch, text_hidden, sampled, outputs)
|
||||
|
||||
ids, hidden, text_step = state.model.mtp_calls[0]
|
||||
assert ids.tolist() == [7, _EOS, 9]
|
||||
# The frame's hidden is this step's last-token hidden of each span.
|
||||
assert torch.equal(hidden, text_hidden[[2, 3, 4]])
|
||||
# The next step adds its own text step.
|
||||
assert not text_step.any()
|
||||
codes = outputs["codes"]["audio"]
|
||||
assert codes[2].tolist() == [7, 8, 9]
|
||||
assert codes[3].tolist() == [_EOS, _EOS + 1, _EOS + 2]
|
||||
assert codes[4].tolist() == [9, 10, 11]
|
||||
assert not codes[:2].any()
|
||||
# Valid only when both the sampled and (for decode) the input CB0 are codec ids.
|
||||
assert outputs["meta"]["codec_frame_valid"].tolist() == [0, 0, 1, 0, 0]
|
||||
assert state._eager_ready == {0: "prefill", 1: "decode", 2: "ended"}
|
||||
assert torch.equal(state._eager_embeds[0], torch.full((_EAGER_DIM,), 107.0))
|
||||
assert state._eager_rows is None
|
||||
|
||||
|
||||
def test_eager_mtp_ignores_rows_recorded_for_another_batch():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0")
|
||||
state._eager_rows = (_EagerBatch([1]), [(0, 0, "r0", False)], torch.tensor([5]))
|
||||
outputs = _eager_outputs(1)
|
||||
state.run_eager_mtp(_EagerBatch([1]), torch.zeros(1, _EAGER_DIM), torch.tensor([[3]]), outputs)
|
||||
assert state.model.mtp_calls == []
|
||||
assert not outputs["codes"]["audio"].any()
|
||||
|
||||
|
||||
def test_eager_mtp_requires_retained_frame_outputs():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0")
|
||||
batch = _EagerBatch([1])
|
||||
state._eager_rows = (batch, [(0, 0, "r0", False)], torch.tensor([5]))
|
||||
with pytest.raises(RuntimeError, match="requires retained codes.audio"):
|
||||
state.run_eager_mtp(batch, torch.zeros(1, _EAGER_DIM), torch.tensor([[3]]), {"codes": {"audio": None}})
|
||||
|
||||
|
||||
def test_eager_decode_input_is_previous_frame_plus_text_step():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0", "r1")
|
||||
state._eager_ready = {0: "r0", 1: "r1"}
|
||||
state._eager_embeds[0] = 5.0
|
||||
state._eager_embeds[1] = 7.0
|
||||
embeds = torch.zeros((2, _EAGER_DIM))
|
||||
batches = [
|
||||
(0, 0, (torch.zeros(_EAGER_DIM), torch.full((_EAGER_DIM,), 2.0))),
|
||||
(1, 1, (None, torch.ones(_EAGER_DIM))),
|
||||
]
|
||||
state._eager_state._apply_eager_frames(batches, embeds, _EagerBatch([1, 1]), None)
|
||||
assert embeds.tolist() == [[7.0] * _EAGER_DIM, [8.0] * _EAGER_DIM]
|
||||
# The deferred MTP must not run for an eager decode row.
|
||||
assert state.model.mtp_calls == []
|
||||
|
||||
|
||||
def test_eager_decode_without_a_frame_fails_loudly():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0")
|
||||
batches = [(0, 0, (torch.zeros(_EAGER_DIM), torch.zeros(_EAGER_DIM)))]
|
||||
with pytest.raises(RuntimeError, match="frame missing"):
|
||||
state._eager_state._apply_eager_frames(batches, torch.zeros((1, _EAGER_DIM)), _EagerBatch([1]), None)
|
||||
|
||||
|
||||
def test_run_preprocess_records_rows_that_keep_a_sample():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "chunk", "final", "decode")
|
||||
state._eager_ready = {2: "decode"}
|
||||
state._eager_embeds[2] = 4.0
|
||||
text_step = torch.ones(_EAGER_DIM)
|
||||
|
||||
def preprocess(input_ids, input_embeds, **info):
|
||||
updates = {"mtp_inputs": (torch.zeros(_EAGER_DIM), text_step)} if input_ids.shape[0] == 1 else {}
|
||||
return input_ids, input_embeds, updates
|
||||
|
||||
state.model.preprocess = preprocess
|
||||
batch = _EagerBatch([3, 3, 1])
|
||||
model_inputs = {"input_ids": torch.zeros(7, dtype=torch.long), "inputs_embeds": torch.zeros((7, _EAGER_DIM))}
|
||||
req_states = SimpleNamespace(
|
||||
prompt_len=np.array([10, 10, 10], dtype=np.int32),
|
||||
num_computed_tokens=np.array([0, 7, 10], dtype=np.int32),
|
||||
)
|
||||
state.run_preprocess(batch, model_inputs, req_states)
|
||||
|
||||
recorded_batch, entries, _ids = state._eager_rows
|
||||
assert recorded_batch is batch
|
||||
# A non-final prefill chunk samples nothing that is kept.
|
||||
assert entries == [(1, 1, "final", True), (2, 2, "decode", False)]
|
||||
assert model_inputs["inputs_embeds"][6].tolist() == [5.0] * _EAGER_DIM
|
||||
assert state.model.mtp_calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_was_valid", [False, True])
|
||||
def test_first_audio_requirement_survives_eos_only_if_audio_was_queued(first_was_valid):
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0")
|
||||
state._first_audio_requests.add("r0")
|
||||
state._eager_state._first_audio_valid = torch.tensor([first_was_valid, False, False, False])
|
||||
batch = _EagerBatch([1])
|
||||
state._eager_rows = (batch, [(0, 0, "r0", False)], torch.tensor([5]))
|
||||
outputs = _eager_outputs(1)
|
||||
outputs["meta"]["first_audio"] = torch.zeros(1, dtype=torch.bool)
|
||||
state.run_eager_mtp(batch, torch.zeros(1, _EAGER_DIM), torch.tensor([[_EOS]]), outputs)
|
||||
assert outputs["meta"]["first_audio"].tolist() == [first_was_valid]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accepted", [[], ["r1"], ["r0", "r1"]])
|
||||
def test_first_audio_marker_requires_accepted_delivery(monkeypatch, accepted):
|
||||
from contextlib import nullcontext
|
||||
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0", "r1")
|
||||
state.model.first_frame_decoder = SimpleNamespace(
|
||||
sample_rate=24000, decode=lambda codes: torch.ones(codes.shape[0], 2)
|
||||
)
|
||||
state._first_audio_stream = SimpleNamespace(wait_stream=lambda stream: None)
|
||||
state._first_audio_sender = SimpleNamespace(submit=lambda ids, pcm, sr, valid: accepted)
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", lambda device: None)
|
||||
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
|
||||
monkeypatch.setattr(torch.Tensor, "record_stream", lambda tensor, stream: None)
|
||||
batch = _EagerBatch([1, 1])
|
||||
state._eager_rows = (batch, [(0, 0, "r0", True), (1, 1, "r1", True)], torch.zeros(2, dtype=torch.long))
|
||||
outputs = _eager_outputs(2)
|
||||
outputs["meta"]["first_audio"] = torch.zeros(2, dtype=torch.bool)
|
||||
|
||||
state.run_eager_mtp(batch, torch.zeros(2, _EAGER_DIM), torch.tensor([[7], [8]]), outputs)
|
||||
|
||||
assert state._first_audio_requests == set(accepted)
|
||||
# A missing route must leave the normal codec path responsible for frame 0;
|
||||
# otherwise it skips the frame and the orchestrator waits forever for it.
|
||||
assert outputs["meta"]["first_audio"].tolist() == ["r0" in accepted, "r1" in accepted]
|
||||
Reference in new issue
Block a user