Jared WenandHongsheng Liu
6e35bfbbae
Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
292 lines
9.8 KiB
Python
292 lines
9.8 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""
|
|
HuggingFace (transformers + diffusers) baseline benchmark for GLM-Image.
|
|
|
|
Supports T2I and I2I modes with the prompt.json dataset.
|
|
Downloads source images for I2I from image_url on first run and caches locally.
|
|
|
|
Usage:
|
|
# T2I mode (text-to-image, no source images needed)
|
|
python benchmarks/glm_image/huggingface/inference.py \
|
|
--model-path zai-org/GLM-Image \
|
|
--mode t2i --num-prompts 10
|
|
|
|
# I2I mode (image-to-image, downloads source images)
|
|
python benchmarks/glm_image/huggingface/inference.py \
|
|
--model-path zai-org/GLM-Image \
|
|
--mode i2i --num-prompts 10
|
|
|
|
# With custom prompt.json
|
|
python benchmarks/glm_image/huggingface/inference.py \
|
|
--model-path zai-org/GLM-Image \
|
|
--mode i2i --dataset-path prompts.json --num-prompts 5
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import requests
|
|
import torch
|
|
from PIL import Image
|
|
|
|
BENCHMARK_DIR = Path(__file__).resolve().parent.parent
|
|
DEFAULT_PROMPT_JSON = BENCHMARK_DIR / "prompt" / "prompt.json"
|
|
IMAGE_CACHE_DIR = BENCHMARK_DIR / "prompt" / "images"
|
|
|
|
DATASET_REPO = "JaredforReal/glm-image-bench"
|
|
DATASET_FILE = "prompt.json"
|
|
|
|
|
|
def _ensure_prompt_json(dataset_path: str | None) -> str:
|
|
"""Return path to prompt.json, downloading from HuggingFace if needed."""
|
|
if dataset_path:
|
|
return dataset_path
|
|
local = DEFAULT_PROMPT_JSON
|
|
if local.exists():
|
|
return str(local)
|
|
print(f"Downloading {DATASET_FILE} from {DATASET_REPO} ...")
|
|
try:
|
|
from huggingface_hub import hf_hub_download
|
|
|
|
downloaded = hf_hub_download(
|
|
repo_id=DATASET_REPO,
|
|
filename=DATASET_FILE,
|
|
repo_type="dataset",
|
|
)
|
|
local.parent.mkdir(parents=True, exist_ok=True)
|
|
import shutil
|
|
|
|
shutil.copy2(downloaded, local)
|
|
print(f"Saved to {local}")
|
|
except ImportError:
|
|
url = f"https://huggingface.co/datasets/{DATASET_REPO}/resolve/main/{DATASET_FILE}"
|
|
import urllib.request
|
|
|
|
local.parent.mkdir(parents=True, exist_ok=True)
|
|
urllib.request.urlretrieve(url, local)
|
|
print(f"Saved to {local}")
|
|
return str(local)
|
|
|
|
|
|
HEIGHT = 1024
|
|
WIDTH = 1024
|
|
SEED = 42
|
|
NUM_INFERENCE_STEPS = 50
|
|
GUIDANCE_SCALE = 1.5
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dataset
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def load_dataset(
|
|
dataset_path: str | None,
|
|
mode: str,
|
|
num_prompts: int,
|
|
) -> list[dict]:
|
|
"""Load prompts from prompt.json and prepare per-request data."""
|
|
path = _ensure_prompt_json(dataset_path)
|
|
with open(path, encoding="utf-8") as f:
|
|
raw = json.load(f)
|
|
|
|
items = []
|
|
for entry in raw:
|
|
if mode == "t2i":
|
|
prompt_key = "t2i_prompt"
|
|
else:
|
|
prompt_key = "i2i_prompt"
|
|
|
|
prompt_text = entry.get(prompt_key, "").strip()
|
|
if not prompt_text:
|
|
continue
|
|
|
|
item = {"prompt": prompt_text}
|
|
if mode == "i2i":
|
|
item["image_url"] = entry.get("image_url", "")
|
|
items.append(item)
|
|
|
|
if num_prompts and len(items) > num_prompts:
|
|
items = items[:num_prompts]
|
|
return items
|
|
|
|
|
|
def download_image(url: str, cache_dir: Path) -> str:
|
|
"""Download an image to cache_dir and return the local path."""
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
fname = url.rsplit("/", 1)[-1]
|
|
local_path = cache_dir / fname
|
|
if local_path.exists():
|
|
return str(local_path)
|
|
print(f" Downloading {url} ...")
|
|
resp = requests.get(url, timeout=30)
|
|
resp.raise_for_status()
|
|
local_path.write_bytes(resp.content)
|
|
return str(local_path)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Benchmark
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def benchmark(args: argparse.Namespace) -> None:
|
|
from diffusers.pipelines.glm_image import GlmImagePipeline
|
|
|
|
print("=" * 60)
|
|
print("GLM-Image HuggingFace Baseline Benchmark")
|
|
print(f"Mode: {args.mode} | Model: {args.model_path}")
|
|
print(f"Size: {args.height}x{args.width} | Steps: {args.num_inference_steps}")
|
|
print("=" * 60)
|
|
|
|
# Load dataset
|
|
items = load_dataset(args.dataset_path, args.mode, args.num_prompts)
|
|
if not items:
|
|
print("No prompts loaded. Exiting.")
|
|
return
|
|
print(f"Loaded {len(items)} prompts for {args.mode} mode")
|
|
|
|
# Download I2I source images
|
|
if args.mode == "i2i":
|
|
print("Preparing source images...")
|
|
for item in items:
|
|
url = item.get("image_url", "")
|
|
if url:
|
|
item["image_path"] = download_image(url, IMAGE_CACHE_DIR)
|
|
else:
|
|
item["image_path"] = None
|
|
|
|
# Load pipeline
|
|
print(f"\nLoading pipeline from {args.model_path} ...")
|
|
t0 = time.perf_counter()
|
|
pipe = GlmImagePipeline.from_pretrained(
|
|
args.model_path,
|
|
torch_dtype=torch.bfloat16,
|
|
device_map="cuda",
|
|
)
|
|
init_time = time.perf_counter() - t0
|
|
print(f"Pipeline loaded in {init_time:.2f}s")
|
|
|
|
# Create output dir
|
|
os.makedirs(args.output_dir, exist_ok=True)
|
|
|
|
# Run benchmark
|
|
generator = torch.Generator(device="cuda").manual_seed(args.seed)
|
|
latencies = []
|
|
success = 0
|
|
failed = 0
|
|
|
|
print(f"\nRunning {len(items)} requests sequentially...")
|
|
print("-" * 60)
|
|
|
|
for i, item in enumerate(items):
|
|
prompt = item["prompt"]
|
|
gen_kwargs: dict = {
|
|
"prompt": prompt,
|
|
"height": args.height,
|
|
"width": args.width,
|
|
"num_inference_steps": args.num_inference_steps,
|
|
"guidance_scale": args.guidance_scale,
|
|
"generator": generator,
|
|
}
|
|
|
|
if args.mode == "i2i":
|
|
img_path = item.get("image_path")
|
|
if img_path and os.path.exists(img_path):
|
|
gen_kwargs["image"] = [Image.open(img_path).convert("RGB")]
|
|
else:
|
|
print(f" [{i + 1}] SKIP: no source image")
|
|
failed += 1
|
|
continue
|
|
|
|
t_start = time.perf_counter()
|
|
try:
|
|
result = pipe(**gen_kwargs)
|
|
image = result.images[0]
|
|
elapsed = time.perf_counter() - t_start
|
|
latencies.append(elapsed)
|
|
success += 1
|
|
|
|
out_path = os.path.join(args.output_dir, f"{i:04d}.png")
|
|
image.save(out_path)
|
|
print(f" [{i + 1}/{len(items)}] {elapsed:.3f}s -> {out_path}")
|
|
except Exception as e:
|
|
elapsed = time.perf_counter() - t_start
|
|
failed += 1
|
|
print(f" [{i + 1}/{len(items)}] FAILED ({elapsed:.3f}s): {e}")
|
|
|
|
# Report
|
|
total_gen_time = sum(latencies) if latencies else 0
|
|
print("\n" + "=" * 60)
|
|
print("HuggingFace Baseline Results")
|
|
print("=" * 60)
|
|
print(f"{'Mode:':<40} {args.mode}")
|
|
print(f"{'Model:':<40} {args.model_path}")
|
|
print(f"{'Image size:':<40} {args.height}x{args.width}")
|
|
print(f"{'Num inference steps:':<40} {args.num_inference_steps}")
|
|
print("-" * 50)
|
|
print(f"{'Pipeline init time (s):':<40} {init_time:.2f}")
|
|
print(f"{'Successful:':<40} {success}/{len(items)}")
|
|
print(f"{'Failed:':<40} {failed}")
|
|
print("-" * 50)
|
|
if latencies:
|
|
arr = np.array(latencies)
|
|
print(f"{'Total generation time (s):':<40} {total_gen_time:.2f}")
|
|
print(f"{'Throughput (img/s):':<40} {success / total_gen_time:.4f}")
|
|
print(f"{'Latency Mean (s):':<40} {arr.mean():.4f}")
|
|
print(f"{'Latency Median (s):':<40} {np.median(arr):.4f}")
|
|
print(f"{'Latency P95 (s):':<40} {np.percentile(arr, 95):.4f}")
|
|
print(f"{'Latency P99 (s):':<40} {np.percentile(arr, 99):.4f}")
|
|
|
|
print(f"\n{'Output dir:':<40} {args.output_dir}")
|
|
print("=" * 60)
|
|
|
|
# Save metrics JSON
|
|
metrics = {
|
|
"backend": "huggingface",
|
|
"mode": args.mode,
|
|
"model": args.model_path,
|
|
"height": args.height,
|
|
"width": args.width,
|
|
"num_inference_steps": args.num_inference_steps,
|
|
"init_time_s": init_time,
|
|
"completed_requests": success,
|
|
"failed_requests": failed,
|
|
"total_gen_time_s": total_gen_time,
|
|
"throughput_qps": success / total_gen_time if total_gen_time > 0 else 0,
|
|
"latency_mean": float(np.mean(latencies)) if latencies else 0,
|
|
"latency_median": float(np.median(latencies)) if latencies else 0,
|
|
"latency_p95": float(np.percentile(latencies, 95)) if latencies else 0,
|
|
"latency_p99": float(np.percentile(latencies, 99)) if latencies else 0,
|
|
}
|
|
if args.output_file:
|
|
with open(args.output_file, "w") as f:
|
|
json.dump(metrics, f, indent=2)
|
|
print(f"Metrics saved to {args.output_file}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="GLM-Image HuggingFace baseline benchmark")
|
|
parser.add_argument("--model-path", type=str, default="zai-org/GLM-Image")
|
|
parser.add_argument("--mode", type=str, default="t2i", choices=["t2i", "i2i"])
|
|
parser.add_argument("--dataset-path", type=str, default=None, help="Path to prompt.json")
|
|
parser.add_argument("--num-prompts", type=int, default=10)
|
|
parser.add_argument("--height", type=int, default=HEIGHT)
|
|
parser.add_argument("--width", type=int, default=WIDTH)
|
|
parser.add_argument("--num-inference-steps", type=int, default=NUM_INFERENCE_STEPS)
|
|
parser.add_argument("--guidance-scale", type=float, default=GUIDANCE_SCALE)
|
|
parser.add_argument("--seed", type=int, default=SEED)
|
|
parser.add_argument("--output-dir", type=str, default="benchmarks/glm_image/huggingface/outputs")
|
|
parser.add_argument("--output-file", type=str, default=None, help="JSON file for metrics")
|
|
args = parser.parse_args()
|
|
benchmark(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|