[CI][Bugfix] Fix the error in generating the performance data table and add a fallback mechanism that prevents the result file from being generated when test case execution fails. (#2839)
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
This commit is contained in:
1 parent
bbd6a443ed
commit
a3ecde98e0
9 files changed
+394
-31
No files matched your search
@@ -0,0 +1,86 @@
|
||||
[
|
||||
{
|
||||
"test_name": null,
|
||||
"backend": null,
|
||||
"timestamp": null,
|
||||
"server_params": {
|
||||
"model": null,
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": false
|
||||
}
|
||||
},
|
||||
"benchmark_params": {
|
||||
"name": null,
|
||||
"dataset": null,
|
||||
"task": null,
|
||||
"width": 0,
|
||||
"height": 0,
|
||||
"num-inference-steps": 0,
|
||||
"num-prompts": 0,
|
||||
"max-concurrency": 0,
|
||||
"num-input-images": 0,
|
||||
"enable-negative-prompt": false,
|
||||
"baseline": {
|
||||
"throughput_qps": 0,
|
||||
"latency_mean": 0,
|
||||
"peak_memory_mb_max": 0,
|
||||
"peak_memory_mb_mean": 0
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"duration": 0,
|
||||
"completed_requests": 0,
|
||||
"failed_requests": 0,
|
||||
"throughput_qps": 0,
|
||||
"latency_mean": 0,
|
||||
"latency_median": 0,
|
||||
"latency_p99": 0,
|
||||
"latency_p95": 0,
|
||||
"latency_p50": 0,
|
||||
"peak_memory_mb_max": 0,
|
||||
"peak_memory_mb_mean": 0,
|
||||
"peak_memory_mb_median": 0,
|
||||
"stage_durations_mean": {},
|
||||
"stage_durations_p50": {},
|
||||
"stage_durations_p99": {},
|
||||
"backend": null,
|
||||
"model": null,
|
||||
"dataset": null,
|
||||
"task": null
|
||||
},
|
||||
"log_file": null,
|
||||
"Model": null,
|
||||
"Framework": null,
|
||||
"Hardware": null,
|
||||
"Deployment": null,
|
||||
"Task": null,
|
||||
"Dataset": null,
|
||||
"resolution": null,
|
||||
"Parallelism": null,
|
||||
"max_concurrency": 0,
|
||||
"Cache": null,
|
||||
"Quantization": null,
|
||||
"offload": null,
|
||||
"compile": null,
|
||||
"Attn_backend": null,
|
||||
"num_inference_steps": 0,
|
||||
"completed": 0,
|
||||
"failed": 0,
|
||||
"throughput_qps": 0,
|
||||
"latency_mean": 0,
|
||||
"latency_median": 0,
|
||||
"latency_p99": 0,
|
||||
"latency_p95": 0,
|
||||
"latency_p50": 0,
|
||||
"peak_memory_mb_max": 0,
|
||||
"peak_memory_mb_mean": 0,
|
||||
"peak_memory_mb_median": 0,
|
||||
"stage_durations_mean": {},
|
||||
"stage_durations_p50": {},
|
||||
"stage_durations_p99": {},
|
||||
"commit_sha": null,
|
||||
"build_id": null,
|
||||
"build_url": null,
|
||||
"source_file": null
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"date": null,
|
||||
"endpoint_type": null,
|
||||
"backend": null,
|
||||
"label": null,
|
||||
"model_id": null,
|
||||
"tokenizer_id": null,
|
||||
"num_prompts": 0,
|
||||
"request_rate": null,
|
||||
"burstiness": 0,
|
||||
"max_concurrency": 0,
|
||||
"duration": 0,
|
||||
"completed": 0,
|
||||
"failed": 0,
|
||||
"total_input_tokens": 0,
|
||||
"total_output_tokens": 0,
|
||||
"request_throughput": 0,
|
||||
"request_goodput": null,
|
||||
"output_throughput": 0,
|
||||
"total_token_throughput": 0,
|
||||
"total_audio_duration_s": 0,
|
||||
"total_audio_frames": 0,
|
||||
"audio_throughput": 0,
|
||||
"max_output_tokens_per_s": 0,
|
||||
"max_concurrent_requests": 0,
|
||||
"rtfx": 0,
|
||||
"mean_ttft_ms": 0,
|
||||
"median_ttft_ms": 0,
|
||||
"p99_ttft_ms": 0,
|
||||
"mean_tpot_ms": 0,
|
||||
"median_tpot_ms": 0,
|
||||
"p99_tpot_ms": 0,
|
||||
"mean_itl_ms": 0,
|
||||
"median_itl_ms": 0,
|
||||
"p99_itl_ms": 0,
|
||||
"mean_e2el_ms": 0,
|
||||
"median_e2el_ms": 0,
|
||||
"p99_e2el_ms": 0,
|
||||
"mean_audio_rtf": 0,
|
||||
"median_audio_rtf": 0,
|
||||
"p99_audio_rtf": 0,
|
||||
"mean_audio_ttfp_ms": 0,
|
||||
"median_audio_ttfp_ms": 0,
|
||||
"p99_audio_ttfp_ms": 0,
|
||||
"mean_audio_duration_s": 0,
|
||||
"median_audio_duration_s": 0,
|
||||
"p99_audio_duration_s": 0,
|
||||
"baseline": {
|
||||
"mean_ttft_ms": 0,
|
||||
"mean_audio_ttfp_ms": 0,
|
||||
"mean_audio_rtf": 0
|
||||
},
|
||||
"random_input_len": 0,
|
||||
"random_output_len": 0
|
||||
}
|
||||
@@ -45,6 +45,8 @@ if CONFIG_FILE_PATH is None:
|
||||
CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE
|
||||
|
||||
BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
|
||||
OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json"
|
||||
|
||||
|
||||
STAGE_CONFIGS_DIR = Path(__file__).parent.parent / "stage_configs"
|
||||
test_params = create_unique_server_params(BENCHMARK_CONFIGS, STAGE_CONFIGS_DIR)
|
||||
@@ -64,7 +66,7 @@ def omni_server(request):
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
server_args = ["--stage-init-timeout", "300", "--init-timeout", "900"]
|
||||
server_args = ["--stage-init-timeout", "600", "--init-timeout", "900"]
|
||||
if stage_config_path:
|
||||
server_args = ["--stage-configs-path", stage_config_path] + server_args
|
||||
with OmniServer(model, server_args) as server:
|
||||
@@ -141,8 +143,17 @@ def run_benchmark(
|
||||
result_dir = "./"
|
||||
|
||||
result_path = os.path.join(result_dir, result_filename)
|
||||
with open(result_path, encoding="utf-8") as f:
|
||||
result = json.load(f)
|
||||
if not os.path.exists(result_path):
|
||||
with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
|
||||
template_result: dict[str, Any] = json.load(f)
|
||||
Path(result_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(result_path, "w", encoding="utf-8") as f:
|
||||
json.dump(template_result, f, ensure_ascii=False, indent=2)
|
||||
print(f"Benchmark result file not generated, fallback to template: {result_path}")
|
||||
result = template_result
|
||||
else:
|
||||
with open(result_path, encoding="utf-8") as f:
|
||||
result = json.load(f)
|
||||
|
||||
if baseline_config:
|
||||
result["baseline"] = _baseline_thresholds_for_step(
|
||||
|
||||
@@ -52,6 +52,7 @@ BENCHMARK_SCRIPT = str(
|
||||
_SESSION_TIMESTAMP = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
_RESULT_LOCK = threading.Lock()
|
||||
_BRANCHPOINT_COMMIT_SHA: str | None = None
|
||||
DIFFUSION_RESULT_TEMPLATE_PATH = Path(__file__).parent / "diffusion_result_template.json"
|
||||
|
||||
|
||||
def _get_config_file_from_argv() -> str | None:
|
||||
@@ -563,10 +564,17 @@ def run_benchmark(
|
||||
|
||||
if process.returncode != 0:
|
||||
tmp_result_file.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"Benchmark script exited with code {process.returncode}")
|
||||
print(f"ERROR:Benchmark script exited with code {process.returncode}")
|
||||
|
||||
if not tmp_result_file.exists():
|
||||
raise FileNotFoundError(f"Benchmark result file not found: {tmp_result_file}")
|
||||
with open(DIFFUSION_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
|
||||
template_payload = json.load(f)
|
||||
# Template schema is fixed and owned by this repo:
|
||||
# ``diffusion_result_template.json`` is a one-item list and metrics live at [0]["result"].
|
||||
template_metrics: dict[str, Any] = template_payload[0]["result"]
|
||||
with open(tmp_result_file, "w", encoding="utf-8") as f:
|
||||
json.dump(template_metrics, f, ensure_ascii=False, indent=2)
|
||||
print(f"Benchmark result file not generated, fallback to template: {tmp_result_file}")
|
||||
|
||||
try:
|
||||
with open(tmp_result_file, encoding="utf-8") as f:
|
||||
|
||||
Reference in new issue
Block a user