vllm-omni/tests/dfx/perf/tests/test_voxcpm2.json

[
  {
    "test_name": "test_voxcpm2",
    "mark": [
      {
        "hardware_marks": {
          "res": {
            "cuda": "H100"
          },
          "num_cards": 1
        }
      },
      "local_model",
      "tts"
    ],
    "server_params": {
      "model": "openbmb/VoxCPM2",
      "trust_remote_code": true,
      "extra_cli_args": [
        "--trust-remote-code"
      ]
    },
    "benchmark_params": [
      {
        "task": "voice_clone",
        "eval_phase": "latency",
        "dataset_name": "seed-tts",
        "backend": "openai-audio-speech",
        "endpoint": "/v1/audio/speech",
        "dataset_path": "linyueqian/seed-tts-eval-subset",
        "num_prompts": [
          20
        ],
        "max_concurrency": [
          1
        ],
        "seed_tts_locale": "en",
        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "H100": {
            "median_audio_ttfp_ms": [
              137.645
            ],
            "median_audio_rtf": [
              0.0497
            ]
          }
        }
      },
      {
        "task": "voice_clone",
        "eval_phase": "throughput",
        "dataset_name": "seed-tts",
        "backend": "openai-audio-speech",
        "endpoint": "/v1/audio/speech",
        "dataset_path": "linyueqian/seed-tts-eval-subset",
        "num_prompts": [
          80
        ],
        "max_concurrency": [
          8
        ],
        "seed_tts_locale": "en",
        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "H100": {
            "median_audio_ttfp_ms": [
              475.3357
            ],
            "median_audio_rtf": [
              0.1243
            ],
            "audio_throughput": [
              36.6358
            ]
          }
        }
      },
      {
        "task": "voice_clone",
        "eval_phase": "quality",
        "enabled": false,
        "dataset_name": "seed-tts",
        "backend": "openai-audio-speech",
        "endpoint": "/v1/audio/speech",
        "dataset_path": "linyueqian/seed-tts-eval-subset",
        "num_prompts": [
          200
        ],
        "max_concurrency": [
          4
        ],
        "seed_tts_locale": "en",
        "seed_tts_wer_eval": true,
        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "H100": {
            "mean_audio_rtf": [
              0.45
            ]
          }
        }
      },
      {
        "task": "default_voice",
        "eval_phase": "latency",
        "dataset_name": "seed-tts-text",
        "backend": "openai-audio-speech",
        "endpoint": "/v1/audio/speech",
        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
        "num_prompts": [
          20
        ],
        "max_concurrency": [
          1
        ],
        "seed_tts_locale": "en",
        "extra_body": {
          "voice": "default"
        },
        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "H100": {
            "median_audio_ttfp_ms": [
              79.6927
            ],
            "median_audio_rtf": [
              0.0423
            ]
          }
        }
      },
      {
        "task": "default_voice",
        "eval_phase": "throughput",
        "dataset_name": "seed-tts-text",
        "backend": "openai-audio-speech",
        "endpoint": "/v1/audio/speech",
        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
        "num_prompts": [
          80
        ],
        "max_concurrency": [
          8
        ],
        "seed_tts_locale": "en",
        "extra_body": {
          "voice": "default"
        },
        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "H100": {
            "median_audio_ttfp_ms": [
              262.6212
            ],
            "median_audio_rtf": [
              0.0753
            ],
            "audio_throughput": [
              103.5843
            ]
          }
        }
      },
      {
        "task": "default_voice",
        "eval_phase": "stress",
        "dataset_name": "seed-tts-text",
        "backend": "openai-audio-speech",
        "endpoint": "/v1/audio/speech",
        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
        "num_prompts": [
          100
        ],
        "request_rate": [
          2.0
        ],
        "seed_tts_locale": "en",
        "extra_body": {
          "voice": "default"
        },
        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "H100": {
            "median_audio_ttfp_ms": [
              80.1095
            ],
            "median_audio_rtf": [
              0.0479
            ],
            "audio_throughput": [
              17.2918
            ]
          }
        }
      },
      {
        "task": "default_voice",
        "eval_phase": "quality",
        "enabled": false,
        "dataset_name": "seed-tts-text",
        "backend": "openai-audio-speech",
        "endpoint": "/v1/audio/speech",
        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
        "num_prompts": [
          200
        ],
        "max_concurrency": [
          4
        ],
        "seed_tts_locale": "en",
        "extra_body": {
          "voice": "default"
        },
        "seed_tts_wer_eval": true,
        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
        "baseline": {
          "H100": {
            "mean_audio_rtf": [
              0.35
            ]
          }
        }
      }
    ]
  }
]