[CI/Build] Fix Qwen3-Omni Talker benchmark workload (#8057)

Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
psv666 authored and GitHub committed 2026-09-24 23:56:53 +08:00
1 parent 0295eeb6f3
commit 2d00d86e5e
5 files changed
+214 -11

No files matched your search

+76
View File
@@ -2045,3 +2045,79 @@ def test_get_samples_forwards_upstream_multimodal_backends_kwarg(mocker: MockerF
# No upstream kwargs: unchanged legacy delegate call.
assert patch.get_samples(args, sentinel) == ["delegated"]
assert calls[-1] == (args, sentinel, {})
@pytest.mark.asyncio
async def test_benchmark_preserves_stage_metrics_request_order_and_missing_snapshots(monkeypatch):
"""Persist compact formal-request snapshots in input order, excluding warmups and retaining gaps."""
second_finished = asyncio.Event()
completion_order = []
# Request 1 carries the empty dict that openai-chat-omni initializes before any SSE merge.
snapshots = [
{"1": {"num_tokens_out": 1536, "finish_reason": "length", "vllm_itls_ms": [8.0, 9.0]}},
{},
{"1": {"num_tokens_out": 486, "finish_reason": "stop"}, "2": {"audio_frames": 24000, "audio_duration_s": 1.0}},
]
expected = [
{"1": {"num_tokens_out": 1536, "finish_reason": "length"}},
None,
{"1": {"num_tokens_out": 486, "finish_reason": "stop"}, "2": {"audio_frames": 24000, "audio_duration_s": 1.0}},
]
async def request_func(request_func_input, session, pbar=None):
request_id = request_func_input.request_id
if not request_id:
return MixRequestFuncOutput(success=True, stage_metrics={"1": {"num_tokens_out": 999}})
index = int(request_id)
if index == 0:
await second_finished.wait()
elif index == 1:
second_finished.set()
completion_order.append(index)
return MixRequestFuncOutput(
success=index != 1,
stage_metrics=snapshots[index],
prompt_len=1,
output_tokens=900,
ttft=0.01,
text_latency=0.1,
latency=0.1,
)
monkeypatch.setitem(patch.ASYNC_REQUEST_FUNCS, "test-stage-metrics", request_func)
result = await patch.benchmark(
task_type=patch.TaskType.GENERATION,
endpoint_type="test-stage-metrics",
api_url="http://unused/v1/chat/completions",
base_url="http://unused",
model_id="test-model",
model_name="test-model",
tokenizer=None,
input_requests=[
patch.SampleRequest(prompt="hello", prompt_len=1, expected_output_len=900, request_id=str(i))
for i in range(3)
],
logprobs=None,
request_rate=float("inf"),
burstiness=1.0,
disable_tqdm=True,
num_warmups=2,
profile=False,
selected_percentile_metrics=[],
selected_percentiles=[],
ignore_eos=True,
goodput_config_dict={},
max_concurrency=3,
lora_modules=None,
extra_headers=None,
extra_body={"return_stage_metrics": True},
ready_check_timeout_sec=0,
)
assert completion_order.index(1) < completion_order.index(0)
assert result["request_stage_metrics"] == expected
@pytest.mark.parametrize("snapshot", [None, {}, "not-a-dict"])
def test_compact_request_stage_metrics_drops_empty_snapshots(snapshot):
"""Empty chat-omni snapshots must not make every result persist request_stage_metrics."""
assert patch._compact_request_stage_metrics(snapshot) is None