[CI/Build] Fix Qwen3-Omni Talker benchmark workload (#8057)

Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
psv666 authored and GitHub committed 2026-09-24 23:56:53 +08:00
1 parent 0295eeb6f3
commit 2d00d86e5e
5 files changed
+214 -11

No files matched your search

@@ -1,6 +1,7 @@
[
{
"test_name": "test_qwen3_omni",
"description": "random row: fixed workload (Thinker 900 / Talker 1536 tokens, min_tokens == max_tokens) for comparable audio RTF; H100 baseline pending recalibration, see #8100",
"mark": [
{
"hardware_marks": {
@@ -43,17 +44,28 @@
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_audio_rtf": [
0.169,
0.2041,
0.2382,
0.3177,
0.4595
]
}
}
"seed": 0,
"random_range_ratio": 0.0,
"extra_body": {
"return_stage_metrics": true,
"sampling_params_list": [
{
"temperature": 0.0,
"min_tokens": 900,
"max_tokens": 900,
"ignore_eos": true
},
{
"temperature": 0.9,
"top_k": 50,
"repetition_penalty": 1.05,
"seed": 0,
"min_tokens": 1536,
"max_tokens": 1536
}
]
},
"baseline": {}
},
{
"dataset_name": "random-mm",
@@ -979,3 +979,58 @@ def test_omni_tpot_baseline_rejects_missing_or_nonfinite_sample(num_tpot_samples
{"baseline": {"H100": {"mean_tpot_ms": 20.0}}},
1,
)
# Mirrors the Thinker/Talker lengths in test_qwen3_omni_no_async_chunk.json.
_FIXED_SAMPLING = {
"sampling_params_list": [{"min_tokens": 900, "max_tokens": 900}, {"min_tokens": 1536, "max_tokens": 1536}]
}
_FIXED_OK = {"0": {"num_tokens_out": 900}, "1": {"num_tokens_out": 1536}}
@pytest.mark.parametrize("extra_body_key", ["extra_body", "extra-body"])
def test_fixed_stage_workload_accepts_exact_lengths(extra_body_key, capsys):
from tests.dfx.perf.scripts.run_benchmark import assert_result
assert_result(
{"completed": 2, "failed": 0, "request_stage_metrics": [_FIXED_OK, _FIXED_OK]},
{extra_body_key: _FIXED_SAMPLING},
2,
)
assert "Fixed stage workload OK: 2 requests, num_tokens_out stage 0=900, stage 1=1536" in capsys.readouterr().out
@pytest.mark.parametrize(
"snapshots",
[
None,
[],
[_FIXED_OK],
[_FIXED_OK, None],
[_FIXED_OK, {}],
[_FIXED_OK, {"0": {"num_tokens_out": 900}, "1": {}}],
[_FIXED_OK, {"0": {"num_tokens_out": 900}, "1": {"num_tokens_out": 486}}],
[_FIXED_OK, {"0": {"num_tokens_out": 900}, "1": {"num_tokens_out": 1537}}],
[_FIXED_OK, {"1": {"num_tokens_out": 1536}}],
[_FIXED_OK, {"0": {"num_tokens_out": 512}, "1": {"num_tokens_out": 1536}}],
],
)
def test_fixed_stage_workload_rejects_missing_or_wrong_lengths(snapshots):
from tests.dfx.perf.scripts.run_benchmark import assert_result
with pytest.raises(AssertionError, match="Fixed stage workload"):
assert_result({"completed": 2, "request_stage_metrics": snapshots}, {"extra_body": _FIXED_SAMPLING}, 2)
def test_fixed_stage_workload_rejects_request_failures():
from tests.dfx.perf.scripts.run_benchmark import assert_result
with pytest.raises(AssertionError, match="Request failures"):
assert_result({"completed": 1, "failed": 1}, {}, 2)
@pytest.mark.parametrize("extra_body", [{}, {"sampling_params_list": [{"max_tokens": 900}]}])
def test_variable_stage_workload_does_not_require_stage_metrics(extra_body):
from tests.dfx.perf.scripts.run_benchmark import assert_result
assert_result({"completed": 2}, {"extra_body": extra_body}, 2)