vllm-omni/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json

[
  {
    "test_name": "test_qwen3_omni_chunk",
    "mark": [
      {
        "hardware_marks": {
          "res": {
            "cuda": "H100"
          },
          "num_cards": 2
        }
      },
      "full_model",
      "omni"
    ],
    "server_params": {
      "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
      "extra_cli_args": [
        "--async-chunk"
      ]
    },
    "benchmark_params": [
      {
        "dataset_name": "random",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          4,
          16,
          32,
          64,
          128
        ],
        "max_concurrency": [
          1,
          4,
          8,
          16,
          32
        ],
        "random_input_len": 2500,
        "random_output_len": 900,
        "ignore_eos": true,
        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "mean_ttft_ms": [
            135.4379,
            187.0896,
            400,
            435.675,
            724.6836
          ],
          "mean_e2el_ms": [
            17619.9851,
            34811.8335,
            43656.9501,
            56657.1548,
            85097.6016
          ],
          "mean_audio_rtf": [
            0.1279,
            0.2238,
            0.2735,
            0.3712,
            0.5634
          ]
        }
      },
      {
        "dataset_name": "random-mm",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          10
        ],
        "request_rate": [
          0.1
        ],
        "random_input_len": 100,
        "random_output_len": 100,
        "random_range_ratio": 0.0,
        "ignore_eos": true,
        "random_mm_base_items_per_request": 1,
        "random_mm_num_mm_items_range_ratio": 0.5,
        "random_mm_limit_mm_per_prompt": {
          "audio": 1
        },
        "random_mm_bucket_config": {
          "(0, 60, 3)": 1.0
        },
        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "mean_ttft_ms": [
            340.329
          ],
          "mean_e2el_ms": [
            4305.477
          ],
          "mean_audio_rtf": [
            0.1305
          ]
        }
      },
      {
        "dataset_name": "random-mm",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          40
        ],
        "request_rate": [
          0.5
        ],
        "random_input_len": 100,
        "random_output_len": 100,
        "random_range_ratio": 0.0,
        "ignore_eos": true,
        "random_mm_base_items_per_request": 2,
        "random_mm_num_mm_items_range_ratio": 0.5,
        "random_mm_limit_mm_per_prompt": {
          "image": 1,
          "video": 1
        },
        "random_mm_bucket_config": {
          "(256, 256, 1)": 0.5,
          "(720, 1280, 2)": 0.5
        },
        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "mean_ttft_ms": [
            220.0909
          ],
          "mean_e2el_ms": [
            5475.9565
          ],
          "mean_audio_rtf": [
            0.1818
          ]
        }
      },
      {
        "dataset_name": "random-mm",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          100
        ],
        "request_rate": [
          1.0
        ],
        "random_input_len": 100,
        "random_output_len": 100,
        "random_range_ratio": 0.0,
        "ignore_eos": true,
        "random_mm_base_items_per_request": 3,
        "random_mm_num_mm_items_range_ratio": 0.5,
        "random_mm_limit_mm_per_prompt": {
          "image": 1,
          "video": 1,
          "audio": 1
        },
        "random_mm_bucket_config": {
          "(256, 256, 1)": 0.34,
          "(720, 1280, 2)": 0.33,
          "(0, 60, 3)": 0.33
        },
        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "mean_ttft_ms": [
            450.0155
          ],
          "mean_e2el_ms": [
            8516.8316
          ],
          "mean_audio_rtf": [
            0.2717
          ]
        }
      },
      {
        "dataset_name": "random",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          4,
          16,
          32,
          64,
          128
        ],
        "max_concurrency": [
          1,
          4,
          8,
          16,
          32
        ],
        "random_input_len": 2500,
        "random_output_len": 900,
        "ignore_eos": true,
        "extra_body": {
          "modalities": [
            "text"
          ]
        },
        "percentile-metrics": "ttft,tpot,itl,e2el",
        "baseline": {
          "mean_ttft_ms": [
            78.4278,
            195.8037,
            333.7318,
            561.805,
            1038.043
          ],
          "mean_e2el_ms": [
            4288.3029,
            7429.9429,
            9451.6184,
            12607.5292,
            18153.2547
          ]
        }
      },
      {
        "dataset_name": "random-mm",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          10
        ],
        "request_rate": [
          0.1
        ],
        "random_input_len": 100,
        "random_output_len": 100,
        "random_range_ratio": 0.0,
        "ignore_eos": true,
        "extra_body": {
          "modalities": [
            "text"
          ]
        },
        "random_mm_base_items_per_request": 1,
        "random_mm_num_mm_items_range_ratio": 0.5,
        "random_mm_limit_mm_per_prompt": {
          "audio": 1
        },
        "random_mm_bucket_config": {
          "(0, 60, 3)": 1.0
        },
        "percentile-metrics": "ttft,tpot,itl,e2el",
        "baseline": {
          "mean_ttft_ms": [
            275.3875
          ],
          "mean_e2el_ms": [
            730.6995
          ]
        }
      },
      {
        "dataset_name": "random-mm",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          40
        ],
        "request_rate": [
          0.5
        ],
        "random_input_len": 100,
        "random_output_len": 100,
        "random_range_ratio": 0.0,
        "ignore_eos": true,
        "extra_body": {
          "modalities": [
            "text"
          ]
        },
        "random_mm_base_items_per_request": 2,
        "random_mm_num_mm_items_range_ratio": 0.5,
        "random_mm_limit_mm_per_prompt": {
          "image": 1,
          "video": 1
        },
        "random_mm_bucket_config": {
          "(256, 256, 1)": 0.5,
          "(720, 1280, 2)": 0.5
        },
        "percentile-metrics": "ttft,tpot,itl,e2el",
        "baseline": {
          "mean_ttft_ms": [
            166.2201
          ],
          "mean_e2el_ms": [
            673.5789
          ]
        }
      },
      {
        "dataset_name": "random-mm",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          100
        ],
        "request_rate": [
          1.0
        ],
        "random_input_len": 100,
        "random_output_len": 100,
        "random_range_ratio": 0.0,
        "ignore_eos": true,
        "extra_body": {
          "modalities": [
            "text"
          ]
        },
        "random_mm_base_items_per_request": 3,
        "random_mm_num_mm_items_range_ratio": 0.5,
        "random_mm_limit_mm_per_prompt": {
          "image": 1,
          "video": 1,
          "audio": 1
        },
        "random_mm_bucket_config": {
          "(256, 256, 1)": 0.34,
          "(720, 1280, 2)": 0.33,
          "(0, 60, 3)": 0.33
        },
        "percentile-metrics": "ttft,tpot,itl,e2el",
        "baseline": {
          "mean_ttft_ms": [
            357.7781
          ],
          "mean_e2el_ms": [
            953.3493
          ]
        }
      }
    ]
  }
]