[CI][MiniCPM-o]MiniCPM-o ACC PERF CI (#5575)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
434c220da1
commit
a5eb2a9a87
50 files changed
+1756
-102
No files matched your search
@@ -6,13 +6,21 @@ Unit tests for patch.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
|
||||
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput, async_request_openai_chat_omni_completions
|
||||
from vllm_omni.benchmarks.patch.patch import (
|
||||
MixRequestFuncOutput,
|
||||
async_request_openai_chat_omni_completions,
|
||||
async_request_openai_realtime_duplex,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
||||
|
||||
@@ -40,6 +48,156 @@ class MockResponse:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch):
|
||||
class FakeRealtimeClient:
|
||||
last_instance = None
|
||||
|
||||
def __init__(self, url):
|
||||
assert url == "ws://localhost:8000/v1/realtime?duplex=1"
|
||||
self.events = RealtimeEventCollector()
|
||||
self.configure_kwargs = None
|
||||
self.sent = []
|
||||
self.response_count = 0
|
||||
self.ack_count = 0
|
||||
type(self).last_instance = self
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return None
|
||||
|
||||
async def configure(self, model, **kwargs):
|
||||
assert model == "openbmb/MiniCPM-o-4_5"
|
||||
self.configure_kwargs = kwargs
|
||||
|
||||
async def send(self, event):
|
||||
self.sent.append(event)
|
||||
if event["type"] != "response.create":
|
||||
return
|
||||
self.response_count += 1
|
||||
response_id = f"resp-{self.response_count}"
|
||||
now = time.monotonic()
|
||||
self.events.add(
|
||||
{"type": "response.created", "response": {"id": response_id}},
|
||||
received_at_s=now + 0.01,
|
||||
)
|
||||
self.events.add(
|
||||
{
|
||||
"type": "response.audio_transcript.delta",
|
||||
"response_id": response_id,
|
||||
"delta": f"turn {self.response_count}",
|
||||
},
|
||||
received_at_s=now + 0.02,
|
||||
)
|
||||
self.events.add(
|
||||
{
|
||||
"type": "response.audio.delta",
|
||||
"response_id": response_id,
|
||||
"delta": base64.b64encode(b"\x00\x00" * 2400).decode(),
|
||||
"sample_rate_hz": 24_000,
|
||||
"metadata": {"audio_duration_ms": 100},
|
||||
},
|
||||
received_at_s=now + 0.03,
|
||||
)
|
||||
self.events.add(
|
||||
{"type": "response.done", "response": {"id": response_id}},
|
||||
received_at_s=now + 0.04,
|
||||
)
|
||||
|
||||
async def acknowledge_playback(self):
|
||||
self.ack_count += 1
|
||||
|
||||
async def close_session(self, **_kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.benchmarks.patch.patch.RealtimeDuplexClient",
|
||||
FakeRealtimeClient,
|
||||
)
|
||||
request_input = RequestFuncInput(
|
||||
model="openbmb/MiniCPM-o-4_5",
|
||||
model_name="openbmb/MiniCPM-o-4_5",
|
||||
prompt="hello",
|
||||
api_url="http://localhost:8000/v1/realtime",
|
||||
prompt_len=1,
|
||||
output_len=20,
|
||||
logprobs=None,
|
||||
multi_modal_content=None,
|
||||
ignore_eos=False,
|
||||
extra_body={"save_duplex_request_metrics": True},
|
||||
)
|
||||
request_input.seed_tts_speech_extra = {"ref_audio": "data:audio/wav;base64,AAAA"}
|
||||
request_input.seed_tts_system_prompt = "Speak exactly."
|
||||
request_input.seed_tts_turns = tuple(
|
||||
SimpleNamespace(utterance_id=f"utt-{index}", target_text=f"text {index}") for index in range(4)
|
||||
)
|
||||
|
||||
output = await async_request_openai_realtime_duplex(
|
||||
request_input,
|
||||
session=None,
|
||||
)
|
||||
|
||||
client = FakeRealtimeClient.last_instance
|
||||
assert client.configure_kwargs["native_duplex"] is False
|
||||
assert client.configure_kwargs["extra_body"] == {
|
||||
"ref_audio": "data:audio/wav;base64,AAAA",
|
||||
}
|
||||
assert client.sent == [
|
||||
event
|
||||
for index in range(4)
|
||||
for event in (
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": f"text {index}"}],
|
||||
},
|
||||
},
|
||||
{"type": "response.create"},
|
||||
)
|
||||
]
|
||||
assert client.ack_count == 4
|
||||
assert output.success is True
|
||||
assert output.generated_text == "turn 1 turn 2 turn 3 turn 4"
|
||||
assert output.audio_duration == pytest.approx(0.4)
|
||||
assert output.ttft > 0
|
||||
assert output.audio_ttfp > output.ttft
|
||||
assert output.audio_rtf > 0
|
||||
assert output.latency > 0
|
||||
assert output.tts_turn_pcm_bytes == [b"\x00\x00" * 2400] * 4
|
||||
assert output.tts_output_pcm_bytes == b"\x00\x00" * 9600
|
||||
session_id = output.duplex_request_metrics[0]["session_id"]
|
||||
assert len(output.duplex_request_metrics) == 4
|
||||
for index, metrics in enumerate(output.duplex_request_metrics):
|
||||
assert metrics == {
|
||||
"session_id": session_id,
|
||||
"request_index": index,
|
||||
"utterance_id": f"utt-{index}",
|
||||
"response_id": f"resp-{index + 1}",
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "conversation.item.create client send to first non-empty text delta",
|
||||
"ttfp": "conversation.item.create client send to first audio packet",
|
||||
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
"ttft_ms": pytest.approx(20.0, abs=2.0),
|
||||
"ttfp_ms": pytest.approx(30.0, abs=2.0),
|
||||
"rtf": pytest.approx(0.3, abs=0.03),
|
||||
"audio_generation_ms": pytest.approx(30.0, abs=2.0),
|
||||
"audio_duration_ms": 100.0,
|
||||
}
|
||||
assert output.duplex_session_metrics == {
|
||||
"session_id": session_id,
|
||||
"audio_turn_count": 4,
|
||||
"mean_ttft_ms": pytest.approx(20.0, abs=2.0),
|
||||
"mean_ttfp_ms": pytest.approx(30.0, abs=2.0),
|
||||
"mean_rtf": pytest.approx(0.3, abs=0.03),
|
||||
}
|
||||
|
||||
|
||||
def create_sse_chunk(data_dict):
|
||||
"""Helper to create SSE formatted chunk"""
|
||||
return f"data: {json.dumps(data_dict)}\n\n".encode()
|
||||
|
||||
@@ -42,7 +42,10 @@ from benchmarks.accuracy.text_to_image.gbench import (
|
||||
summarize_generated_records as summarize_gebench_generated_records,
|
||||
summarize_gebench_results,
|
||||
)
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import seed_tts_bench_argv
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_serve_common_argv,
|
||||
seed_tts_bench_argv,
|
||||
)
|
||||
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
|
||||
|
||||
@@ -57,6 +60,25 @@ def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
|
||||
assert argv[dataset_idx + 1] == "zhaochenyang20/seed-tts-eval"
|
||||
|
||||
|
||||
def test_build_serve_common_argv_accepts_realtime_seed_tts_route(tmp_path):
|
||||
argv = build_serve_common_argv(
|
||||
host="127.0.0.1",
|
||||
port=8000,
|
||||
model="openbmb/MiniCPM-o-4_5",
|
||||
num_prompts=4,
|
||||
max_concurrency=1,
|
||||
num_warmups=0,
|
||||
percentile_metrics="ttft,audio_ttfp,audio_rtf",
|
||||
result_dir=tmp_path,
|
||||
result_filename="result.json",
|
||||
backend="openai-realtime-tts",
|
||||
endpoint="/v1/realtime",
|
||||
)
|
||||
|
||||
assert argv[argv.index("--backend") + 1] == "openai-realtime-tts"
|
||||
assert argv[argv.index("--endpoint") + 1] == "/v1/realtime"
|
||||
|
||||
|
||||
def test_sync_dataset_env_preserves_seed_tts_hf_repo_id(monkeypatch):
|
||||
ns = argparse.Namespace(
|
||||
daily_omni_repo=None,
|
||||
|
||||
@@ -28,10 +28,13 @@ if _MODULE_NAME not in sys.modules:
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( # noqa: E402
|
||||
SeedTTSDataset,
|
||||
SeedTTSDesignDataset,
|
||||
SeedTTSDesignSampleRequest,
|
||||
SeedTTSSampleRequest,
|
||||
SeedTTSTextDataset,
|
||||
SeedTTSTextSampleRequest,
|
||||
SeedTTSTurn,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -70,6 +73,94 @@ def mock_tokenizer(mocker):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_seed_tts_dataset_groups_four_turns_with_one_reference(seed_tts_root, mock_tokenizer):
|
||||
ds = SeedTTSDataset(
|
||||
dataset_path=str(seed_tts_root),
|
||||
random_seed=0,
|
||||
locale="en",
|
||||
disable_shuffle=True,
|
||||
)
|
||||
|
||||
requests = ds.sample(
|
||||
mock_tokenizer,
|
||||
num_requests=1,
|
||||
turns_per_session=4,
|
||||
no_oversample=True,
|
||||
)
|
||||
|
||||
assert len(requests) == 1
|
||||
request = requests[0]
|
||||
assert [turn.utterance_id for turn in request.seed_tts_turns] == [
|
||||
"utt000",
|
||||
"utt001",
|
||||
"utt002",
|
||||
"utt003",
|
||||
]
|
||||
assert [turn.target_text for turn in request.seed_tts_turns] == [
|
||||
"target text 0",
|
||||
"target text 1",
|
||||
"target text 2",
|
||||
"target text 3",
|
||||
]
|
||||
assert request.seed_tts_speech_extra["ref_text"] == "ref text 0"
|
||||
assert request.seed_tts_ref_wav_path.endswith("utt000.wav")
|
||||
|
||||
|
||||
def test_seed_tts_eval_expands_grouped_pcm_by_turn():
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
_expand_seed_tts_turn_outputs,
|
||||
)
|
||||
|
||||
request = SeedTTSSampleRequest(
|
||||
prompt="target 0",
|
||||
prompt_len=4,
|
||||
expected_output_len=100,
|
||||
multi_modal_data=None,
|
||||
request_id="session-0",
|
||||
seed_tts_turns=tuple(
|
||||
SeedTTSTurn(utterance_id=f"utt-{index}", target_text=f"target {index}") for index in range(4)
|
||||
),
|
||||
)
|
||||
output = types.SimpleNamespace(
|
||||
success=True,
|
||||
tts_turn_pcm_bytes=[bytes([index]) for index in range(4)],
|
||||
)
|
||||
|
||||
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
|
||||
|
||||
assert [item.prompt for item in requests] == [f"target {index}" for index in range(4)]
|
||||
assert [item.seed_tts_utterance_id for item in requests] == [f"utt-{index}" for index in range(4)]
|
||||
assert [item.tts_output_pcm_bytes for item in outputs] == [bytes([index]) for index in range(4)]
|
||||
|
||||
|
||||
def test_seed_tts_eval_keeps_session_pcm_for_single_turn_chat():
|
||||
"""Chat-omni Seed-TTS sets session PCM only; expand must not wipe it."""
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
_expand_seed_tts_turn_outputs,
|
||||
)
|
||||
|
||||
request = SeedTTSSampleRequest(
|
||||
prompt="target 0",
|
||||
prompt_len=4,
|
||||
expected_output_len=100,
|
||||
multi_modal_data=None,
|
||||
request_id="session-0",
|
||||
seed_tts_turns=(SeedTTSTurn(utterance_id="utt-0", target_text="target 0"),),
|
||||
)
|
||||
pcm = b"\x01\x02\x03\x04"
|
||||
output = types.SimpleNamespace(
|
||||
success=True,
|
||||
tts_output_pcm_bytes=pcm,
|
||||
tts_turn_pcm_bytes=None,
|
||||
)
|
||||
|
||||
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
|
||||
|
||||
assert len(requests) == 1
|
||||
assert requests[0].prompt == "target 0"
|
||||
assert outputs[0].tts_output_pcm_bytes == pcm
|
||||
|
||||
|
||||
def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mock_tokenizer):
|
||||
ds = SeedTTSTextDataset(
|
||||
dataset_path=str(seed_tts_root),
|
||||
|
||||
@@ -128,6 +128,17 @@ def assert_result(
|
||||
request_rate: Any | None = None,
|
||||
) -> None:
|
||||
assert result["completed"] == num_prompt, "Request failures exist"
|
||||
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
|
||||
if expected_audio_turns is not None:
|
||||
session_metrics = result.get("duplex_session_metrics")
|
||||
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
|
||||
assert len(session_metrics) == num_prompt, (
|
||||
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
|
||||
)
|
||||
assert all(
|
||||
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
|
||||
for metric in session_metrics
|
||||
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
|
||||
if not assert_baseline:
|
||||
return
|
||||
baseline_data = params.get("baseline", {})
|
||||
@@ -140,13 +151,13 @@ def assert_result(
|
||||
request_rate=request_rate,
|
||||
)
|
||||
if "throughput" in metric_name:
|
||||
if current_value <= baseline_value:
|
||||
print(
|
||||
f"ERROR: Throughput test results were below baseline: {metric_name}: {current_value} > {baseline_value}"
|
||||
)
|
||||
assert current_value >= baseline_value, (
|
||||
f"Throughput test result was below baseline: {metric_name}: {current_value} < {baseline_value}"
|
||||
)
|
||||
else:
|
||||
if current_value >= baseline_value:
|
||||
print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}")
|
||||
assert current_value <= baseline_value, (
|
||||
f"Test result exceeded baseline: {metric_name}: {current_value} > {baseline_value}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@@ -201,6 +212,8 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
"expected_duplex_audio_turns_per_session",
|
||||
"baseline_hardware",
|
||||
}
|
||||
|
||||
for key, value in params.items():
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
128,
|
||||
256
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
16
|
||||
],
|
||||
"no_oversample": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"request_throughput": [
|
||||
0.7803,
|
||||
1.1008,
|
||||
1.2027,
|
||||
1.3254
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
166.9764,
|
||||
307.7133,
|
||||
323.584,
|
||||
356.6077
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1361.1682,
|
||||
3800.6332,
|
||||
6961.8636,
|
||||
12609.4295
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
635.8099,
|
||||
1615.6591,
|
||||
2888.9831,
|
||||
8320.5422
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.3151,
|
||||
0.8568,
|
||||
1.5425,
|
||||
2.8224
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,65 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_duplex_seed_tts",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5_duplex.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"num_prompts": [
|
||||
1
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"no_oversample": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"seed_tts_turns_per_session": 4,
|
||||
"expected_duplex_audio_turns_per_session": 4,
|
||||
"extra_body": {
|
||||
"save_duplex_request_metrics": true
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"request_throughput": [
|
||||
0.2415
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
122.6731
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4073.804
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
523.4768
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.28295
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -285,3 +285,69 @@ def test_create_paired_omni_benchmark_pytest_params(tmp_path):
|
||||
assert bench_row == ("test_tts", 0)
|
||||
assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks)
|
||||
assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("metric", "current", "baseline"),
|
||||
[
|
||||
("request_throughput", 1.0, 1.0),
|
||||
("mean_ttft_ms", 100.0, 100.0),
|
||||
],
|
||||
)
|
||||
def test_omni_baseline_accepts_values_at_threshold(metric, current, baseline):
|
||||
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||
|
||||
assert_result(
|
||||
{"completed": 1, metric: current},
|
||||
{"baseline": {metric: baseline}},
|
||||
1,
|
||||
assert_baseline=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("metric", "current", "baseline", "message"),
|
||||
[
|
||||
("request_throughput", 0.9, 1.0, "below baseline"),
|
||||
("mean_ttft_ms", 101.0, 100.0, "exceeded baseline"),
|
||||
],
|
||||
)
|
||||
def test_omni_baseline_rejects_regressions(metric, current, baseline, message):
|
||||
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||
|
||||
with pytest.raises(AssertionError, match=message):
|
||||
assert_result(
|
||||
{"completed": 1, metric: current},
|
||||
{"baseline": {metric: baseline}},
|
||||
1,
|
||||
assert_baseline=True,
|
||||
)
|
||||
|
||||
|
||||
def test_omni_duplex_expected_audio_turns_accepts_complete_session():
|
||||
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||
|
||||
assert_result(
|
||||
{
|
||||
"completed": 1,
|
||||
"duplex_session_metrics": [{"audio_turn_count": 4}],
|
||||
},
|
||||
{"expected_duplex_audio_turns_per_session": 4},
|
||||
1,
|
||||
assert_baseline=False,
|
||||
)
|
||||
|
||||
|
||||
def test_omni_duplex_expected_audio_turns_rejects_incomplete_session():
|
||||
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||
|
||||
with pytest.raises(AssertionError, match="emitted 4 audio turns"):
|
||||
assert_result(
|
||||
{
|
||||
"completed": 1,
|
||||
"duplex_session_metrics": [{"audio_turn_count": 3}],
|
||||
},
|
||||
{"expected_duplex_audio_turns_per_session": 4},
|
||||
1,
|
||||
assert_baseline=False,
|
||||
)
|
||||
@@ -48,12 +48,12 @@ def _prompt() -> str:
|
||||
return "What is the capital of China? Answer in 20 words."
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True)
|
||||
def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None:
|
||||
"""
|
||||
Test that invalid audio format is properly rejected.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + invalid audio format
|
||||
Output Modal: error
|
||||
"""
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
|
||||
|
||||
Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
|
||||
~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
|
||||
text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
|
||||
+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.accuracy.qwen3_omni import run_qwen_omni_acc_benchmark as _acc_bench
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_acc_benchmark_cli_argv,
|
||||
find_vllm_cli,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.minicpmo_4_5_duplex import SERVER_PARAMS as DUPLEX_TEST_PARAMS
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5")
|
||||
_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml")
|
||||
_RESULT_DIR = Path(
|
||||
os.environ.get(
|
||||
"ACC_BENCH_RESULT_DIR",
|
||||
str(Path(__file__).resolve().parent / "results"),
|
||||
)
|
||||
)
|
||||
|
||||
_MIN_DAILY_OMNI_ACCURACY = 0.78
|
||||
_MAX_SEED_TTS_MEAN_WER = 0.05
|
||||
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
|
||||
_DAILY_EXTRA_BODY = {
|
||||
"modalities": ["text"],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": False,
|
||||
"use_tts_template": True,
|
||||
},
|
||||
}
|
||||
_SEED_EXTRA_BODY = {
|
||||
"modalities": ["text", "audio"],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": False,
|
||||
"use_tts_template": True,
|
||||
},
|
||||
}
|
||||
# Server flags required for MiniCPM interleaved image/audio packs (Daily-Omni only).
|
||||
# Do not reuse these for Seed-TTS: ``--interleave-mm-strings`` + TTS ref_audio can trip
|
||||
# msgspec ValidationError and kill the orchestrator mid-suite.
|
||||
# Also pinned in ``minicpmo_4_5.yaml`` stage 0 so they survive config-path routing.
|
||||
_DAILY_OMNI_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--interleave-mm-strings",
|
||||
"--media-io-kwargs",
|
||||
'{"video":{"fps":1,"num_frames":128}}',
|
||||
]
|
||||
_SEED_TTS_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
daily_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DEPLOY_CONFIG,
|
||||
server_args=list(_DAILY_OMNI_SERVER_ARGS),
|
||||
)
|
||||
]
|
||||
|
||||
seed_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DEPLOY_CONFIG,
|
||||
server_args=list(_SEED_TTS_SERVER_ARGS),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _require_vllm_cli() -> None:
|
||||
try:
|
||||
find_vllm_cli()
|
||||
except FileNotFoundError as exc:
|
||||
pytest.skip(str(exc))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Inline cached Hub/local videos because the test server has no local-media allowlist."""
|
||||
original = _acc_bench.daily_omni_bench_argv
|
||||
|
||||
def _wrapped() -> list[str]:
|
||||
argv = list(original())
|
||||
if "--daily-omni-inline-local-video" not in argv:
|
||||
argv.append("--daily-omni-inline-local-video")
|
||||
return argv
|
||||
|
||||
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("datasets")
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=True,
|
||||
skip_daily=False,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--min-daily-omni-accuracy",
|
||||
str(_MIN_DAILY_OMNI_ACCURACY),
|
||||
"--temperature",
|
||||
"0",
|
||||
"--output-len",
|
||||
"512",
|
||||
"--daily-omni-input-mode",
|
||||
"all",
|
||||
"--daily-omni-pack-mode",
|
||||
"minicpm-interleave",
|
||||
"--daily-extra-body-json",
|
||||
json.dumps(_DAILY_EXTRA_BODY, separators=(",", ":")),
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--max-seed-tts-mean-wer",
|
||||
str(_MAX_SEED_TTS_MEAN_WER),
|
||||
"--seed-extra-body-json",
|
||||
json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")),
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", DUPLEX_TEST_PARAMS, indirect=True)
|
||||
def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
|
||||
"""Gate Seed-TTS WER through the explicit Realtime TTS contract."""
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
num_prompts=50,
|
||||
max_concurrency=1,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--max-seed-tts-mean-wer",
|
||||
str(_MAX_SEED_TTS_MEAN_WER),
|
||||
"--seed-extra-body-json",
|
||||
json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")),
|
||||
"--seed-backend",
|
||||
"openai-realtime-tts",
|
||||
"--seed-endpoint",
|
||||
"/v1/realtime",
|
||||
"--seed-tts-turns-per-session",
|
||||
"4",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
@@ -19,6 +19,7 @@ import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
@@ -121,9 +122,13 @@ def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
|
||||
|
||||
def find_vllm_cli() -> str:
|
||||
exe = shutil.which("vllm")
|
||||
if not exe:
|
||||
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
|
||||
return exe
|
||||
if exe:
|
||||
return exe
|
||||
# Fallback when pytest is invoked via absolute venv path without activating PATH.
|
||||
sibling = Path(sys.executable).resolve().parent / "vllm"
|
||||
if sibling.is_file() and os.access(sibling, os.X_OK):
|
||||
return str(sibling)
|
||||
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
|
||||
|
||||
|
||||
def run_vllm_bench_subprocess(vllm: str, argv: list[str], *, extra_env: dict[str, str] | None = None) -> None:
|
||||
@@ -150,6 +155,11 @@ def build_serve_common_argv(
|
||||
result_dir: Path,
|
||||
result_filename: str,
|
||||
ready_check_timeout_sec: int | None = None,
|
||||
trust_remote_code: bool = False,
|
||||
backend: str = "openai-chat-omni",
|
||||
endpoint: str = "/v1/chat/completions",
|
||||
temperature: float | None = None,
|
||||
output_len: int | None = None,
|
||||
) -> list[str]:
|
||||
out = [
|
||||
"bench",
|
||||
@@ -162,9 +172,9 @@ def build_serve_common_argv(
|
||||
"--model",
|
||||
model,
|
||||
"--endpoint",
|
||||
"/v1/chat/completions",
|
||||
endpoint,
|
||||
"--backend",
|
||||
"openai-chat-omni",
|
||||
backend,
|
||||
"--request-rate",
|
||||
"inf",
|
||||
"--num-prompts",
|
||||
@@ -184,6 +194,12 @@ def build_serve_common_argv(
|
||||
]
|
||||
if ready_check_timeout_sec is not None:
|
||||
out.extend(["--ready-check-timeout-sec", str(int(ready_check_timeout_sec))])
|
||||
if trust_remote_code:
|
||||
out.append("--trust-remote-code")
|
||||
if temperature is not None:
|
||||
out.extend(["--temperature", str(temperature)])
|
||||
if output_len is not None:
|
||||
out.extend(["--output-len", str(int(output_len))])
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -171,7 +171,13 @@ def _preserve_benchmark_dataset_env() -> Any:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[str]:
|
||||
def _build_common_args(
|
||||
ns: argparse.Namespace,
|
||||
*,
|
||||
result_filename: str,
|
||||
backend: str = "openai-chat-omni",
|
||||
endpoint: str = "/v1/chat/completions",
|
||||
) -> list[str]:
|
||||
return build_serve_common_argv(
|
||||
host=ns.host,
|
||||
port=ns.port,
|
||||
@@ -183,6 +189,11 @@ def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[
|
||||
result_dir=ns.result_dir,
|
||||
result_filename=result_filename,
|
||||
ready_check_timeout_sec=ns.ready_check_timeout_sec,
|
||||
trust_remote_code=ns.trust_remote_code,
|
||||
backend=backend,
|
||||
endpoint=endpoint,
|
||||
temperature=getattr(ns, "temperature", None),
|
||||
output_len=getattr(ns, "output_len", None),
|
||||
)
|
||||
|
||||
|
||||
@@ -201,6 +212,9 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
pack_mode = getattr(ns, "daily_omni_pack_mode", None)
|
||||
if pack_mode:
|
||||
argv.extend(["--daily-omni-pack-mode", pack_mode])
|
||||
if ns.daily_omni_save_eval_items:
|
||||
argv.append("--daily-omni-save-eval-items")
|
||||
print("\n$", vllm, *argv, "\n", flush=True)
|
||||
@@ -217,7 +231,12 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
result_filename = f"qwen_omni_acc_seed_tts_{tag}.json"
|
||||
extra = json.loads(ns.seed_extra_body_json)
|
||||
argv = (
|
||||
_build_common_args(ns, result_filename=result_filename)
|
||||
_build_common_args(
|
||||
ns,
|
||||
result_filename=result_filename,
|
||||
backend=ns.seed_backend,
|
||||
endpoint=ns.seed_endpoint,
|
||||
)
|
||||
+ seed_tts_bench_argv(locale=ns.seed_tts_locale)
|
||||
+ [
|
||||
"--seed-tts-wer-eval",
|
||||
@@ -225,6 +244,8 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
if ns.seed_tts_turns_per_session > 1:
|
||||
argv.extend(["--seed-tts-turns-per-session", str(ns.seed_tts_turns_per_session)])
|
||||
if ns.seed_tts_wer_save_items:
|
||||
argv.append("--seed-tts-wer-save-items")
|
||||
if ns.seed_tts_file_ref_audio:
|
||||
@@ -266,6 +287,23 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
help="If set, forwarded to ``vllm bench serve`` (probe first request until success). "
|
||||
"Omit to use upstream default (typically skip).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--trust-remote-code",
|
||||
action="store_true",
|
||||
help="Forward trust_remote_code to ``vllm bench serve`` for models with custom code.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Optional sampling temperature forwarded to ``vllm bench serve`` (e.g. 0 for MiniCPM-o Daily-Omni MCQ).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--output-len",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional max output tokens forwarded to ``vllm bench serve`` (e.g. 512 for MiniCPM-o Daily-Omni).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--result-dir",
|
||||
type=Path,
|
||||
@@ -295,6 +333,13 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
help="Optional local Videos root; if omitted, media is fetched lazily from Hub Videos.tar.",
|
||||
)
|
||||
p.add_argument("--daily-omni-input-mode", choices=("all", "visual", "audio"), default="all")
|
||||
p.add_argument(
|
||||
"--daily-omni-pack-mode",
|
||||
choices=("qwen", "minicpm-interleave"),
|
||||
default=None,
|
||||
help="Daily-Omni multimodal pack recipe. Use ``minicpm-interleave`` for MiniCPM-o "
|
||||
"(1fps interleaved image/audio; see PR #5606).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--daily-extra-body-json",
|
||||
default='{"modalities":["text"]}',
|
||||
@@ -326,6 +371,22 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
default='{"modalities":["text","audio"]}',
|
||||
help="JSON for Seed-TTS chat requests (must include audio for synthesis + PCM capture).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-backend",
|
||||
default="openai-chat-omni",
|
||||
help="Seed-TTS benchmark backend (for example openai-realtime-tts).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-endpoint",
|
||||
default="/v1/chat/completions",
|
||||
help="Seed-TTS serving endpoint corresponding to --seed-backend.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-tts-turns-per-session",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Group this many Seed-TTS target texts into each Realtime session.",
|
||||
)
|
||||
p.add_argument("--seed-tts-wer-save-items", action="store_true")
|
||||
p.add_argument(
|
||||
"--seed-tts-file-ref-audio",
|
||||
@@ -400,6 +461,11 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
min_mean_sim=ns.min_seed_tts_mean_sim,
|
||||
min_mean_utmos=ns.min_seed_tts_mean_utmos,
|
||||
)
|
||||
if ns.seed_tts_turns_per_session > 1:
|
||||
expected_turns = ns.num_prompts * ns.seed_tts_turns_per_session
|
||||
evaluated_turns = int(data.get("seed_tts_content_evaluated") or 0)
|
||||
if evaluated_turns != expected_turns:
|
||||
errs.append(f"seed_tts_content_evaluated={evaluated_turns} != expected {expected_turns}")
|
||||
if errs:
|
||||
failed.extend([f"[Seed-TTS] {e}" for e in errs])
|
||||
else:
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
RealtimeEventCollector,
|
||||
build_realtime_url,
|
||||
read_pcm16_wav,
|
||||
summarize_session_request_metrics,
|
||||
)
|
||||
|
||||
_url_with_model = build_realtime_url
|
||||
@@ -194,6 +195,38 @@ class DemoState:
|
||||
summaries[response_id] = timing
|
||||
return summaries
|
||||
|
||||
def session_request_metrics(
|
||||
self,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return benchmark-style metrics for every response in this session."""
|
||||
requests: list[dict[str, object]] = []
|
||||
for request_index, (response_id, timing) in enumerate(self.response_timing_summaries().items()):
|
||||
metrics = timing.get("request_metrics")
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
requests.append(
|
||||
{
|
||||
"session_id": session_id,
|
||||
"request_index": request_index,
|
||||
"response_id": response_id,
|
||||
**metrics,
|
||||
}
|
||||
)
|
||||
return requests
|
||||
|
||||
def session_metric_summary(
|
||||
self,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Average TTFT, TTFP, and RTF across responses that emitted audio."""
|
||||
return summarize_session_request_metrics(
|
||||
self.session_request_metrics(session_id=session_id),
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
def first_index(self, event_type: str, predicate=None) -> int | None:
|
||||
for index, event in enumerate(self.events):
|
||||
if event.get("type") != event_type:
|
||||
@@ -1523,6 +1556,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
"input_transcription_ok": input_transcription_ok,
|
||||
"completed_response_ids": completed_response_ids,
|
||||
"response_timings": state.response_timing_summaries(),
|
||||
"request_metrics": state.session_request_metrics(session_id=args.session_id),
|
||||
"session_metrics": state.session_metric_summary(session_id=args.session_id),
|
||||
"lifecycle_counts_ok": lifecycle_counts_ok,
|
||||
"validation_mode": validation_mode,
|
||||
"scenario": scenario,
|
||||
|
||||
@@ -8,6 +8,7 @@ import hashlib
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
@@ -19,11 +20,25 @@ SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from minicpmo_realtime_duplex_scenarios import ( # noqa: E402
|
||||
_ref_audio_data_url,
|
||||
_url_with_model,
|
||||
run_demo,
|
||||
)
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _scenario_module():
|
||||
"""Delay heavyweight vLLM imports so ``--help`` remains directly executable."""
|
||||
import minicpmo_realtime_duplex_scenarios
|
||||
|
||||
return minicpmo_realtime_duplex_scenarios
|
||||
|
||||
|
||||
def _ref_audio_data_url(path: str) -> str:
|
||||
return _scenario_module()._ref_audio_data_url(path)
|
||||
|
||||
|
||||
def _url_with_model(*args, **kwargs) -> str:
|
||||
return _scenario_module()._url_with_model(*args, **kwargs)
|
||||
|
||||
|
||||
async def run_demo(args):
|
||||
return await _scenario_module().run_demo(args)
|
||||
|
||||
|
||||
def _with_resume_mode(url: str) -> str:
|
||||
|
||||
@@ -73,7 +73,7 @@ def get_max_batch_size(size_type="few"):
|
||||
def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text-only input generating text output via OpenAI API.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text
|
||||
Output Modal: text
|
||||
Input Setting: stream=False
|
||||
@@ -99,7 +99,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text-only input generating text + audio output via OpenAI API.
|
||||
This exercises Talker TTS region detection and the Code2Wav stage.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -123,7 +123,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test audio input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + audio
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -159,7 +159,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test image input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + image
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -187,7 +187,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test video input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + video
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -216,7 +216,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test multi-modal input (text + audio + video + image) generating text + audio output.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + audio + video + image
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
|
||||
@@ -31,6 +31,29 @@ from vllm_omni.experimental.fullduplex.client import build_realtime_url
|
||||
pytestmark = pytest.mark.omni
|
||||
|
||||
|
||||
def _assert_request_metrics(metrics: object, *, expected_count: int) -> None:
|
||||
assert isinstance(metrics, list)
|
||||
assert len(metrics) == expected_count
|
||||
for request_index, request in enumerate(metrics):
|
||||
assert isinstance(request["session_id"], str)
|
||||
assert request["request_index"] == request_index
|
||||
assert isinstance(request["response_id"], str)
|
||||
assert request["ttft_ms"] is not None and request["ttft_ms"] >= 0
|
||||
assert request["ttfp_ms"] >= 0
|
||||
assert request["rtf"] is not None and request["rtf"] >= 0
|
||||
assert request["audio_generation_ms"] >= 0
|
||||
assert request["audio_duration_ms"] > 0
|
||||
|
||||
|
||||
def _assert_session_metrics(metrics: object, *, expected_count: int) -> None:
|
||||
assert isinstance(metrics, dict)
|
||||
assert isinstance(metrics["session_id"], str)
|
||||
assert metrics["audio_turn_count"] == expected_count
|
||||
assert metrics["mean_ttft_ms"] is not None and metrics["mean_ttft_ms"] >= 0
|
||||
assert metrics["mean_ttfp_ms"] is not None and metrics["mean_ttfp_ms"] >= 0
|
||||
assert metrics["mean_rtf"] is not None and metrics["mean_rtf"] >= 0
|
||||
|
||||
|
||||
async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]:
|
||||
async def receive() -> list[dict[str, object]]:
|
||||
events: list[dict[str, object]] = []
|
||||
@@ -103,22 +126,22 @@ def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
result = asyncio.run(
|
||||
run_demo(
|
||||
demo_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "single_session",
|
||||
)
|
||||
)
|
||||
args = demo_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "single_session",
|
||||
)
|
||||
args.turns = 2
|
||||
result = asyncio.run(run_demo(args))
|
||||
assert result["ok"] is True
|
||||
assert result["audio_delta_count"] > 0
|
||||
assert result["done_count"] == 1
|
||||
assert result["done_count"] == 2
|
||||
assert result["error_count"] == 0
|
||||
assert result["all_audio_responses_have_transcript"] is True
|
||||
assert result["transcript_delta_done_ok"] is True
|
||||
_assert_request_metrics(result["request_metrics"], expected_count=2)
|
||||
_assert_session_metrics(result["session_metrics"], expected_count=2)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -144,3 +167,6 @@ def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str,
|
||||
assert all(session["audio_delta_count"] > 0 for session in result["sessions"])
|
||||
assert all(session["done_count"] == 1 for session in result["sessions"])
|
||||
assert all(session["error_count"] == 0 for session in result["sessions"])
|
||||
for session in result["sessions"]:
|
||||
_assert_request_metrics(session["request_metrics"], expected_count=1)
|
||||
_assert_session_metrics(session["session_metrics"], expected_count=1)
|
||||
@@ -80,7 +80,7 @@ def get_max_batch_size(size_type="few"):
|
||||
def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text + video input generating text output.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + video
|
||||
Output Modal: text
|
||||
Input Setting: stream=False
|
||||
@@ -108,7 +108,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Verify that sequential requests produce independent results and that the
|
||||
second request does not receive the first request's audio or token state.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text (two different prompts)
|
||||
Output Modal: text + audio (both)
|
||||
"""
|
||||
@@ -149,7 +149,7 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text input generating a longer audio output to exercise the
|
||||
Code2Wav stage across multiple frames.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text (longer prompt)
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
|
||||
@@ -799,6 +799,7 @@ def test_realtime_duplex_demo_reads_response_playback_cursor():
|
||||
def test_realtime_duplex_demo_partitions_timing_by_response_identity():
|
||||
demo = _load_demo_module()
|
||||
state = demo.DemoState()
|
||||
state.input_commit_sent_at_s.extend((9.9, 19.9))
|
||||
for response_id, created_at_s, audio_at_s, token_count in (
|
||||
("resp-1", 10.0, 10.1, 3),
|
||||
("resp-2", 20.0, 20.2, 5),
|
||||
@@ -826,15 +827,66 @@ def test_realtime_duplex_demo_partitions_timing_by_response_identity():
|
||||
},
|
||||
received_at_s=audio_at_s,
|
||||
)
|
||||
state.add(
|
||||
{
|
||||
"type": "response.audio_transcript.delta",
|
||||
"response_id": response_id,
|
||||
"delta": "audio",
|
||||
},
|
||||
received_at_s=created_at_s + 0.05,
|
||||
)
|
||||
|
||||
timings = state.response_timing_summaries()
|
||||
requests = state.session_request_metrics(session_id="seed-tts-session")
|
||||
session_metrics = state.session_metric_summary(session_id="seed-tts-session")
|
||||
|
||||
assert timings["resp-1"]["stage0_tokens"]["output_token_count"] == 3
|
||||
assert timings["resp-1"]["audio_output"]["response_created_to_first_audio_ms"] == 100.0
|
||||
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] is None
|
||||
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] == 200.0
|
||||
assert timings["resp-2"]["stage0_tokens"]["output_token_count"] == 5
|
||||
assert timings["resp-2"]["audio_output"]["response_created_to_first_audio_ms"] == 200.0
|
||||
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] is None
|
||||
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] == 300.0
|
||||
assert requests == [
|
||||
{
|
||||
"session_id": "seed-tts-session",
|
||||
"request_index": 0,
|
||||
"response_id": "resp-1",
|
||||
"ttft_ms": 150.0,
|
||||
"ttfp_ms": 200.0,
|
||||
"rtf": 2.5,
|
||||
"audio_generation_ms": 200.0,
|
||||
"audio_duration_ms": 80.0,
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
},
|
||||
{
|
||||
"session_id": "seed-tts-session",
|
||||
"request_index": 1,
|
||||
"response_id": "resp-2",
|
||||
"ttft_ms": 150.0,
|
||||
"ttfp_ms": 300.0,
|
||||
"rtf": 3.75,
|
||||
"audio_generation_ms": 300.0,
|
||||
"audio_duration_ms": 80.0,
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
},
|
||||
]
|
||||
assert session_metrics == {
|
||||
"session_id": "seed-tts-session",
|
||||
"audio_turn_count": 2,
|
||||
"mean_ttft_ms": 150.0,
|
||||
"mean_ttfp_ms": 250.0,
|
||||
"mean_rtf": 3.125,
|
||||
}
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_model_policy_accepts_one_listen_per_streamed_turn():
|
||||
|
||||
@@ -300,6 +300,56 @@ def test_strip_single_engine_args_model_does_not_trigger_warning(mocker):
|
||||
assert "model" not in warned_args
|
||||
|
||||
|
||||
def test_new_format_deploy_yaml_does_not_strip_interleave_cli_overrides(tmp_path):
|
||||
"""New-format deploy YAMLs must not strip interleave_mm_strings / media_io_kwargs.
|
||||
|
||||
Pytest historically passed ``--stage-configs-path`` for deploy YAMLs; stripping
|
||||
those CLI flags dropped MiniCPM interleaved AV packing and cut Daily-Omni
|
||||
accuracy from ~78% to ~70%.
|
||||
"""
|
||||
from vllm_omni.entrypoints.utils import is_new_format_deploy_config
|
||||
|
||||
yaml_path = tmp_path / "minicpm_deploy.yaml"
|
||||
yaml_path.write_text(
|
||||
"""
|
||||
pipeline: minicpmo_4_5
|
||||
async_chunk: true
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
""".strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
path = str(yaml_path)
|
||||
assert is_new_format_deploy_config(path)
|
||||
|
||||
kwargs = {
|
||||
"stage_configs_path": path,
|
||||
"trust_remote_code": True,
|
||||
"interleave_mm_strings": True,
|
||||
"media_io_kwargs": {"video": {"fps": 1, "num_frames": 128}},
|
||||
"gpu_memory_utilization": 0.55,
|
||||
}
|
||||
# Mirror AsyncOmniEngine._resolve_stage_configs strip decision.
|
||||
if kwargs.get("stage_configs_path") is not None and not is_new_format_deploy_config(kwargs["stage_configs_path"]):
|
||||
base = AsyncOmniEngine._strip_single_engine_args(dict(kwargs))
|
||||
else:
|
||||
base = dict(kwargs)
|
||||
assert base.get("interleave_mm_strings") is True
|
||||
assert base.get("media_io_kwargs") == {"video": {"fps": 1, "num_frames": 128}}
|
||||
# Parent EngineArgs fields also remain available for deploy merge.
|
||||
assert base.get("gpu_memory_utilization") == 0.55
|
||||
|
||||
legacy = tmp_path / "legacy.yaml"
|
||||
legacy.write_text("stage_args:\n - stage_id: 0\n", encoding="utf-8")
|
||||
assert not is_new_format_deploy_config(str(legacy))
|
||||
legacy_kwargs = dict(kwargs, stage_configs_path=str(legacy))
|
||||
stripped = AsyncOmniEngine._strip_single_engine_args(legacy_kwargs)
|
||||
assert "interleave_mm_strings" not in stripped
|
||||
assert "media_io_kwargs" not in stripped
|
||||
assert "gpu_memory_utilization" not in stripped
|
||||
|
||||
|
||||
# For https://github.com/vllm-project/vllm-omni/issues/3293
|
||||
def test_tensor_parallel_size_none_is_handled():
|
||||
"""Ensure the tensor parallel size of None isn't forwarded."""
|
||||
|
||||
@@ -51,6 +51,19 @@ from vllm_omni.outputs import OmniRequestOutput
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_native_input_append_supports_explicit_session_opt_out():
|
||||
session = DuplexSession(
|
||||
"sid-explicit-tts",
|
||||
DuplexSessionConfig(model="test-model"),
|
||||
capabilities=DuplexCapabilities.minicpmo45_native(),
|
||||
)
|
||||
|
||||
assert OmniDuplexSessionHandler._uses_native_input_append(session) is True
|
||||
|
||||
session.config.extra_body["minicpmo45_native_duplex"] = False
|
||||
assert OmniDuplexSessionHandler._uses_native_input_append(session) is False
|
||||
|
||||
|
||||
class _ModelConfig:
|
||||
model = "test-model"
|
||||
|
||||
|
||||
@@ -11,6 +11,9 @@ from vllm_omni.experimental.fullduplex.client import (
|
||||
read_pcm16_wav,
|
||||
write_pcm16_wav,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
|
||||
MiniCPMO45DuplexPolicy,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -32,6 +35,19 @@ def test_realtime_client_builds_explicit_native_duplex_url():
|
||||
}
|
||||
|
||||
|
||||
def test_seed_tts_initial_text_is_part_of_native_duplex_context():
|
||||
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
|
||||
"Speak exactly.",
|
||||
True,
|
||||
"The quick brown fox.",
|
||||
)
|
||||
|
||||
assert prefix == "<|im_start|>system\nSpeak exactly.\n<|audio_start|>"
|
||||
assert suffix == (
|
||||
"<|audio_end|><|im_end|>\n<|im_start|>user\nThe quick brown fox.<|im_end|>\n<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
|
||||
def test_realtime_client_builds_resume_only_url_when_autostart_disabled():
|
||||
url = build_realtime_url(
|
||||
"ws://localhost:8099/v1/realtime?duplex=1",
|
||||
@@ -86,6 +102,59 @@ async def test_realtime_client_configure_sends_explicit_ref_audio():
|
||||
assert session["ref_audio"] == "data:audio/wav;base64,AAAA"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_client_configure_sends_seed_tts_text_condition():
|
||||
class Client(RealtimeDuplexClient):
|
||||
def __init__(self):
|
||||
super().__init__("ws://unused")
|
||||
self.sent = []
|
||||
|
||||
async def send(self, event):
|
||||
self.sent.append(event)
|
||||
self.events.add({"type": "session.created"})
|
||||
|
||||
client = Client()
|
||||
|
||||
await client.configure(
|
||||
"openbmb/MiniCPM-o-4_5",
|
||||
instructions="Speak the requested text exactly.",
|
||||
initial_user_text="The quick brown fox.",
|
||||
timeout_s=1,
|
||||
)
|
||||
|
||||
session = client.sent[0]["session"]
|
||||
assert session["instructions"] == "Speak the requested text exactly."
|
||||
assert session["extra_body"]["duplex_initial_user_text"] == "The quick brown fox."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_client_configure_explicit_tts_opts_out_of_native_duplex():
|
||||
class Client(RealtimeDuplexClient):
|
||||
def __init__(self):
|
||||
super().__init__("ws://unused")
|
||||
self.sent = []
|
||||
|
||||
async def send(self, event):
|
||||
self.sent.append(event)
|
||||
self.events.add({"type": "session.created"})
|
||||
|
||||
client = Client()
|
||||
|
||||
await client.configure(
|
||||
"openbmb/MiniCPM-o-4_5",
|
||||
native_duplex=False,
|
||||
auto_response=False,
|
||||
extra_body={"ref_audio": "data:audio/wav;base64,AAAA"},
|
||||
timeout_s=1,
|
||||
)
|
||||
|
||||
session_extra_body = client.sent[0]["session"]["extra_body"]
|
||||
assert session_extra_body == {
|
||||
"ref_audio": "data:audio/wav;base64,AAAA",
|
||||
"minicpmo45_native_duplex": False,
|
||||
}
|
||||
|
||||
|
||||
def test_realtime_event_collector_partitions_audio_by_response():
|
||||
collector = RealtimeEventCollector()
|
||||
collector.add({"type": "response.created", "response": {"id": "resp-a"}})
|
||||
@@ -134,6 +203,26 @@ def test_realtime_event_collector_reports_engine_token_and_audio_intervals():
|
||||
},
|
||||
received_at_s=received_at_s,
|
||||
)
|
||||
collector.add(
|
||||
{
|
||||
"type": "response.audio_transcript.delta",
|
||||
"response_id": "resp-a",
|
||||
"delta": "",
|
||||
},
|
||||
received_at_s=10.1,
|
||||
)
|
||||
collector.add(
|
||||
{
|
||||
"type": "response.audio_transcript.delta",
|
||||
"response_id": "resp-a",
|
||||
"delta": "hello",
|
||||
},
|
||||
received_at_s=10.15,
|
||||
)
|
||||
collector.add(
|
||||
{"type": "response.done", "response": {"id": "resp-a"}},
|
||||
received_at_s=10.4,
|
||||
)
|
||||
|
||||
timing = collector.timing_summary(
|
||||
after_s=10.0,
|
||||
@@ -175,6 +264,19 @@ def test_realtime_event_collector_reports_engine_token_and_audio_intervals():
|
||||
},
|
||||
"max_chunk_gap_ms": 110.0,
|
||||
}
|
||||
assert timing["request_metrics"] == {
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
"ttft_ms": 250.0,
|
||||
"ttfp_ms": 300.0,
|
||||
"rtf": pytest.approx(1.916667),
|
||||
"audio_generation_ms": 460.0,
|
||||
"audio_duration_ms": 240.0,
|
||||
}
|
||||
|
||||
|
||||
def test_response_timing_ignores_unowned_session_level_metrics():
|
||||
|
||||
@@ -19,6 +19,11 @@ DEPLOY_CONFIG = modify_stage_config(
|
||||
get_deploy_config_path("minicpmo_4_5_duplex.yaml"),
|
||||
updates={
|
||||
"base_config": get_deploy_config_path("minicpmo_4_5.yaml"),
|
||||
# Talker context is 4096 (tts_config.max_position_embeddings); KV sizing
|
||||
# is left automatic so duplex matches the simplex deploy profiles.
|
||||
"stages": {
|
||||
1: {"max_model_len": 4096},
|
||||
},
|
||||
},
|
||||
)
|
||||
ASSET_DIR = Path(__file__).resolve().parents[1] / "assets" / "minicpmo_4_5"
|
||||
|
||||
@@ -2881,7 +2881,12 @@ def iter_omni_server(
|
||||
print("OmniServer stopping...")
|
||||
else:
|
||||
if stage_config_path is not None:
|
||||
server_args += ["--stage-configs-path", stage_config_path]
|
||||
from vllm_omni.entrypoints.utils import is_new_format_deploy_config
|
||||
|
||||
if is_new_format_deploy_config(stage_config_path):
|
||||
server_args += ["--deploy-config", stage_config_path]
|
||||
else:
|
||||
server_args += ["--stage-configs-path", stage_config_path]
|
||||
|
||||
with (
|
||||
OmniServer(
|
||||
|
||||
@@ -2164,6 +2164,33 @@ class TestBaseConfigInheritance:
|
||||
class TestPlatformOverrides:
|
||||
"""Test platform-specific deploy config overrides."""
|
||||
|
||||
def test_minicpmo_4_5_cuda_caps_talker_kv_cache(self):
|
||||
pipeline = resolve_pipeline_config("minicpmo_4_5")
|
||||
assert isinstance(pipeline, PipelineConfig)
|
||||
deploy_path = Path(get_deploy_config_path("minicpmo_4_5.yaml"))
|
||||
|
||||
cuda = _apply_platform_overrides(load_deploy_config(deploy_path), platform="cuda")
|
||||
cuda_stages = merge_pipeline_deploy(pipeline, cuda)
|
||||
# Talker KV is left to automatic sizing (no hard kv_cache_memory_bytes cap).
|
||||
assert "kv_cache_memory_bytes" not in cuda_stages[1].yaml_engine_args
|
||||
|
||||
# The CUDA memory budget must not leak into the existing NPU profile.
|
||||
npu = _apply_platform_overrides(load_deploy_config(deploy_path), platform="npu")
|
||||
npu_stages = merge_pipeline_deploy(pipeline, npu)
|
||||
assert "kv_cache_memory_bytes" not in npu_stages[1].yaml_engine_args
|
||||
|
||||
# Multi-GPU replica profiles place Talker on dedicated devices and
|
||||
# also leave KV sizing automatic.
|
||||
for filename in (
|
||||
"minicpmo_4_5_3gpu_stage1_replicas.yaml",
|
||||
"minicpmo_4_5_4gpu_stage1_replicas.yaml",
|
||||
):
|
||||
replica = _apply_platform_overrides(
|
||||
load_deploy_config(Path(get_deploy_config_path(filename))), platform="cuda"
|
||||
)
|
||||
replica_stages = merge_pipeline_deploy(pipeline, replica)
|
||||
assert "kv_cache_memory_bytes" not in replica_stages[1].yaml_engine_args
|
||||
|
||||
def test_npu_overrides(self):
|
||||
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
|
||||
if not deploy_path.exists():
|
||||
|
||||
Reference in new issue
Block a user