[Perf] Qwen-Image Performance Nightly CI test (#1805)
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
df93b3a965
commit
1f5eca5686
13 files changed
+1291
-4
No files matched your search
@@ -246,6 +246,52 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Qwen-Image Diffusion Perf Test with H100"
|
||||
key: nightly-qwen-image-performance
|
||||
timeout_in_minutes: 300
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/perf/results
|
||||
- export CACHE_DIT_VERSION=1.3.0
|
||||
- pytest -s -v tests/perf/scripts/run_diffusion_benchmark.py --config-file tests/perf/tests/test_qwen_image_vllm_omni.json
|
||||
- buildkite-agent artifact upload "tests/perf/results/benchmark_results_*.json"
|
||||
- buildkite-agent artifact upload "tests/perf/results/logs/*.log"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 4
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":email: Nightly Collection & Email"
|
||||
key: nightly-perf-distribution
|
||||
depends_on:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
@@ -109,8 +110,6 @@ async def async_request_chat_completions(
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
try:
|
||||
choices = resp_json.get("choices", [])
|
||||
if choices and isinstance(choices, list):
|
||||
@@ -120,7 +119,8 @@ async def async_request_chat_completions(
|
||||
if content and isinstance(content, list) and len(content) > 0:
|
||||
first_item = content[0]
|
||||
if isinstance(first_item, dict):
|
||||
output.stage_durations = first_item.get("stage_durations")
|
||||
output.stage_durations = first_item.get("stage_durations") or {}
|
||||
output.peak_memory_mb = first_item.get("peak_memory_mb", 0.0)
|
||||
except (IndexError, TypeError, AttributeError):
|
||||
pass
|
||||
else:
|
||||
@@ -335,12 +335,263 @@ async def async_request_v1_videos(
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_image_sglang(
|
||||
input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
output = RequestFuncOutput()
|
||||
output.start_time = time.perf_counter()
|
||||
|
||||
# Check if we need to use multipart (for image edits with input images)
|
||||
if input.image_paths and len(input.image_paths) > 0:
|
||||
# Use multipart/form-data for image edits
|
||||
data = aiohttp.FormData()
|
||||
data.add_field("model", input.model)
|
||||
data.add_field("prompt", input.prompt)
|
||||
data.add_field("response_format", "b64_json")
|
||||
|
||||
if input.width and input.height:
|
||||
data.add_field("size", f"{input.width}x{input.height}")
|
||||
|
||||
# Merge extra parameters
|
||||
for key, value in input.extra_body.items():
|
||||
data.add_field(key, str(value))
|
||||
|
||||
# Add image file(s)
|
||||
for idx, img_path in enumerate(input.image_paths):
|
||||
if os.path.exists(img_path):
|
||||
data.add_field(
|
||||
"image",
|
||||
open(img_path, "rb"),
|
||||
filename=os.path.basename(img_path),
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
else:
|
||||
output.error = f"Image file not found: {img_path}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, data=data) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
else:
|
||||
output.error = f"HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
except Exception as e:
|
||||
output.error = str(e)
|
||||
output.success = False
|
||||
else:
|
||||
# Use JSON for text-to-image generation
|
||||
payload = {
|
||||
"model": input.model,
|
||||
"prompt": input.prompt,
|
||||
"n": 1,
|
||||
"response_format": "b64_json",
|
||||
}
|
||||
|
||||
if input.width and input.height:
|
||||
payload["size"] = f"{input.width}x{input.height}"
|
||||
|
||||
if input.num_inference_steps:
|
||||
payload["num_inference_steps"] = input.num_inference_steps
|
||||
|
||||
payload.update(input.extra_body)
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, json=payload) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
else:
|
||||
output.error = f"HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
except Exception as e:
|
||||
output.error = str(e)
|
||||
output.success = False
|
||||
|
||||
output.latency = time.perf_counter() - output.start_time
|
||||
|
||||
# Check SLO if defined
|
||||
if input.slo_ms is not None and output.success:
|
||||
output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_video_sglang(
|
||||
input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
output = RequestFuncOutput()
|
||||
output.start_time = time.perf_counter()
|
||||
|
||||
# 1. Submit Job
|
||||
job_id = None
|
||||
# Check if we need to upload images (Multipart) or just send JSON
|
||||
if input.image_paths and len(input.image_paths) > 0:
|
||||
# Use multipart/form-data
|
||||
data = aiohttp.FormData()
|
||||
data.add_field("model", input.model)
|
||||
data.add_field("prompt", input.prompt)
|
||||
|
||||
if input.width and input.height:
|
||||
data.add_field("size", f"{input.width}x{input.height}")
|
||||
|
||||
# Add extra body fields to form data if possible, or assume simple key-values
|
||||
# Note: Nested dicts in extra_body might need JSON serialization if API expects it stringified
|
||||
if input.extra_body:
|
||||
data.add_field("extra_body", json.dumps(input.extra_body))
|
||||
|
||||
# Explicitly add fps/num_frames if they are not in extra_body (bench_serving logic overrides)
|
||||
if input.num_frames:
|
||||
data.add_field("num_frames", str(input.num_frames))
|
||||
if input.fps:
|
||||
data.add_field("fps", str(input.fps))
|
||||
|
||||
# Add image file
|
||||
# Currently only support single image upload as 'input_reference' per API spec
|
||||
img_path = input.image_paths[0]
|
||||
if os.path.exists(img_path):
|
||||
data.add_field(
|
||||
"input_reference",
|
||||
open(img_path, "rb"),
|
||||
filename=os.path.basename(img_path),
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
else:
|
||||
output.error = f"Image file not found: {img_path}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, data=data) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
job_id = resp_json.get("id")
|
||||
else:
|
||||
output.error = f"Submit failed HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
except Exception as e:
|
||||
output.error = f"Submit exception: {str(e)}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
else:
|
||||
# Use JSON
|
||||
payload: dict[str, Any] = {
|
||||
"model": input.model,
|
||||
"prompt": input.prompt,
|
||||
}
|
||||
if input.width and input.height:
|
||||
payload["size"] = f"{input.width}x{input.height}"
|
||||
if input.num_frames:
|
||||
payload["num_frames"] = input.num_frames
|
||||
if input.fps:
|
||||
payload["fps"] = input.fps
|
||||
if input.num_inference_steps:
|
||||
payload["num_inference_steps"] = input.num_inference_steps
|
||||
|
||||
payload.update(input.extra_body)
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, json=payload) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
job_id = resp_json.get("id")
|
||||
else:
|
||||
output.error = f"Submit failed HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
except Exception as e:
|
||||
output.error = f"Submit exception: {str(e)}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
if not job_id:
|
||||
output.error = "No job_id returned"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
# 2. Poll for completion
|
||||
# Assuming the API returns a 'status' field.
|
||||
# We construct the check URL. Assuming api_url is like .../v1/videos
|
||||
# The check url should be .../v1/videos/{id}
|
||||
check_url = f"{input.api_url}/{job_id}"
|
||||
|
||||
while True:
|
||||
try:
|
||||
async with session.get(check_url) as response:
|
||||
if response.status == 200:
|
||||
status_data = await response.json()
|
||||
status = status_data.get("status")
|
||||
if status == "completed":
|
||||
output.success = True
|
||||
output.response_body = status_data
|
||||
if "peak_memory_mb" in status_data:
|
||||
output.peak_memory_mb = status_data["peak_memory_mb"]
|
||||
break
|
||||
elif status == "failed":
|
||||
output.success = False
|
||||
output.error = f"Job failed: {status_data.get('error')}"
|
||||
break
|
||||
else:
|
||||
# queued or processing
|
||||
await asyncio.sleep(1.0)
|
||||
else:
|
||||
output.success = False
|
||||
output.error = f"Poll failed HTTP {response.status}: {await response.text()}"
|
||||
break
|
||||
except Exception as e:
|
||||
output.success = False
|
||||
output.error = f"Poll exception: {str(e)}"
|
||||
break
|
||||
|
||||
output.latency = time.perf_counter() - output.start_time
|
||||
|
||||
# Check SLO if defined
|
||||
if input.slo_ms is not None and output.success:
|
||||
output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
backends_function_mapping = {
|
||||
"2i": {
|
||||
"vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
|
||||
"openai": (async_request_openai_images, "/v1/images/generations"),
|
||||
"sglang": (async_request_image_sglang, "/v1/images/generations"),
|
||||
},
|
||||
"2v": {
|
||||
"v1/videos": (async_request_v1_videos, "/v1/videos"),
|
||||
"sglang": (async_request_video_sglang, "/v1/videos"),
|
||||
},
|
||||
}
|
||||
@@ -745,6 +745,15 @@ def calculate_metrics(
|
||||
latencies = [o.latency for o in success_outputs]
|
||||
peak_memories = [o.peak_memory_mb for o in success_outputs if o.peak_memory_mb > 0]
|
||||
|
||||
# Aggregate per-stage durations across all successful requests that reported them.
|
||||
stage_duration_lists: dict[str, list[float]] = {}
|
||||
for o in success_outputs:
|
||||
for stage, duration in (o.stage_durations or {}).items():
|
||||
stage_duration_lists.setdefault(stage, []).append(duration)
|
||||
stage_durations_mean = {s: float(np.mean(v)) for s, v in stage_duration_lists.items()}
|
||||
stage_durations_p50 = {s: float(np.percentile(v, 50)) for s, v in stage_duration_lists.items()}
|
||||
stage_durations_p99 = {s: float(np.percentile(v, 99)) for s, v in stage_duration_lists.items()}
|
||||
|
||||
metrics = {
|
||||
"duration": total_duration,
|
||||
"completed_requests": num_success,
|
||||
@@ -758,6 +767,9 @@ def calculate_metrics(
|
||||
"peak_memory_mb_max": max(peak_memories) if peak_memories else 0,
|
||||
"peak_memory_mb_mean": np.mean(peak_memories) if peak_memories else 0,
|
||||
"peak_memory_mb_median": np.median(peak_memories) if peak_memories else 0,
|
||||
"stage_durations_mean": stage_durations_mean,
|
||||
"stage_durations_p50": stage_durations_p50,
|
||||
"stage_durations_p99": stage_durations_p99,
|
||||
}
|
||||
|
||||
if slo_enabled:
|
||||
@@ -953,6 +965,12 @@ async def benchmark(args):
|
||||
print("{:<40} {:<15.2f}".format("Peak Memory Mean (MB):", metrics["peak_memory_mb_mean"]))
|
||||
print("{:<40} {:<15.2f}".format("Peak Memory Median (MB):", metrics["peak_memory_mb_median"]))
|
||||
|
||||
if metrics["stage_durations_mean"]:
|
||||
print(f"{'-' * 50}")
|
||||
print("Stage Durations Mean (s):")
|
||||
for stage, val in metrics["stage_durations_mean"].items():
|
||||
print("{:<40} {:<15.4f}".format(f" {stage}:", val))
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
||||
if args.output_file:
|
||||
@@ -976,7 +994,7 @@ if __name__ == "__main__":
|
||||
"--backend",
|
||||
type=str,
|
||||
default="vllm-omni",
|
||||
choices=["vllm-omni", "openai", "v1/videos"],
|
||||
choices=["vllm-omni", "openai", "sglang", "v1/videos"],
|
||||
help="Backend to target the benchmark to.",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
"""
|
||||
Performance benchmark CI runner for diffusion models.
|
||||
|
||||
Supports two server backends:
|
||||
- vllm-omni (default): starts DiffusionServer via vllm_omni.entrypoints.cli.main,
|
||||
benchmarks with diffusion_benchmark_serving.py --backend vllm-omni
|
||||
- sglang: starts SglangServer via `sglang serve`,
|
||||
benchmarks with diffusion_benchmark_serving.py --backend sglang
|
||||
|
||||
A config JSON file is REQUIRED via --config-file:
|
||||
pytest run_diffusion_benchmark.py --config-file tests/perf/tests/test_qwen_image_vllm_omni.json
|
||||
pytest run_diffusion_benchmark.py --config-file tests/perf/tests/test_qwen_image_sglang_diffusion.json
|
||||
|
||||
JSON config entries are distinguished by a "server_type" field ("vllm-omni" or "sglang").
|
||||
sglang entries support two additional fields under server_params:
|
||||
- "env": dict of extra environment variables (e.g. SGLANG_CACHE_DIT_ENABLED)
|
||||
- "cache_dit_config": dict written to a temp YAML and passed as
|
||||
--cache-dit-config to sglang serve (requires cache-dit == 1.3.0)
|
||||
|
||||
All benchmark results for a session are consolidated into a single JSON file under
|
||||
BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable).
|
||||
Each entry in the file contains the test metadata (test_name, backend, benchmark_params,
|
||||
timestamp) together with the raw metrics returned by the benchmark script.
|
||||
"""
|
||||
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_RESULT_DIR = Path(__file__).parent.parent / "results"
|
||||
BENCHMARK_RESULT_DIR = Path(os.environ.get("DIFFUSION_BENCHMARK_DIR", str(_DEFAULT_RESULT_DIR)))
|
||||
|
||||
BENCHMARK_SCRIPT = str(
|
||||
Path(__file__).parent.parent.parent.parent / "benchmarks" / "diffusion" / "diffusion_benchmark_serving.py"
|
||||
)
|
||||
|
||||
# Single aggregated result file for the entire benchmark session.
|
||||
# Populated lazily after CONFIG_FILE_PATH is resolved.
|
||||
_SESSION_TIMESTAMP = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
_RESULT_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _get_config_file_from_argv() -> str | None:
|
||||
"""Read --config-file from sys.argv at import time so pytest parametrize can use it.
|
||||
|
||||
pytest_addoption (below) registers the same flag so pytest does not reject it.
|
||||
Supports both ``--config-file path`` and ``--config-file=path`` forms.
|
||||
Returns None if the flag is not present; callers must handle the missing case.
|
||||
"""
|
||||
for i, arg in enumerate(sys.argv):
|
||||
if arg == "--config-file" and i + 1 < len(sys.argv):
|
||||
return sys.argv[i + 1]
|
||||
if arg.startswith("--config-file="):
|
||||
return arg.split("=", 1)[1]
|
||||
return None
|
||||
|
||||
|
||||
CONFIG_FILE_PATH = _get_config_file_from_argv()
|
||||
if CONFIG_FILE_PATH is None:
|
||||
raise ValueError(
|
||||
"--config-file is required. Pass the path to a benchmark config JSON, e.g.:\n"
|
||||
" pytest run_diffusion_benchmark.py "
|
||||
"--config-file tests/perf/tests/test_qwen_image_vllm_omni.json"
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_refs(configs: list[dict[str, Any]], config_dir: Path) -> list[dict[str, Any]]:
|
||||
"""Resolve {"$ref": "filename.json"} in benchmark_params fields."""
|
||||
for cfg in configs:
|
||||
bp = cfg.get("benchmark_params")
|
||||
if isinstance(bp, dict) and "$ref" in bp:
|
||||
ref_path = config_dir / bp["$ref"]
|
||||
try:
|
||||
with open(ref_path, encoding="utf-8") as f:
|
||||
cfg["benchmark_params"] = json.load(f)
|
||||
except FileNotFoundError:
|
||||
raise ValueError(f"benchmark_params $ref not found: {ref_path}")
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"JSON parsing error in {ref_path}: {e}")
|
||||
return configs
|
||||
|
||||
|
||||
def load_configs(config_path: str) -> list[dict[str, Any]]:
|
||||
try:
|
||||
abs_path = Path(config_path).resolve()
|
||||
with open(abs_path, encoding="utf-8") as f:
|
||||
configs = json.load(f)
|
||||
return _resolve_refs(configs, abs_path.parent)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"JSON parsing error: {str(e)}")
|
||||
except FileNotFoundError:
|
||||
raise ValueError(f"Configuration file not found: {config_path}")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to load configuration file: {str(e)}")
|
||||
|
||||
|
||||
BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
|
||||
|
||||
_config_stem = Path(CONFIG_FILE_PATH).stem # e.g. "test_qwen_image_vllm_omni"
|
||||
AGGREGATED_RESULT_FILE = BENCHMARK_RESULT_DIR / f"benchmark_results_{_config_stem}_{_SESSION_TIMESTAMP}.json"
|
||||
|
||||
|
||||
def _append_to_aggregated_file(record: dict[str, Any]) -> None:
|
||||
"""Thread-safe append of *record* to the session-level aggregated JSON file.
|
||||
|
||||
The file contains a JSON array; each call loads the existing array (or
|
||||
starts a new one), appends the record, and writes the file back atomically.
|
||||
"""
|
||||
with _RESULT_LOCK:
|
||||
BENCHMARK_RESULT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if AGGREGATED_RESULT_FILE.exists():
|
||||
with open(AGGREGATED_RESULT_FILE, encoding="utf-8") as f:
|
||||
records: list[dict] = json.load(f)
|
||||
else:
|
||||
records = []
|
||||
records.append(record)
|
||||
with open(AGGREGATED_RESULT_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(records, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
# Register --config-file with pytest so it does not reject the argument.
|
||||
def pytest_addoption(parser: pytest.Parser) -> None:
|
||||
parser.addoption(
|
||||
"--config-file",
|
||||
action="store",
|
||||
default=None,
|
||||
help=(
|
||||
"Path to the benchmark config JSON file (required). "
|
||||
"Example: --config-file tests/perf/tests/test_qwen_image_vllm_omni.json"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_server_lock = threading.Lock()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_open_port() -> int:
|
||||
"""Return an available TCP port on localhost."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("", 0))
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _wait_for_port(host: str, port: int, timeout: int = 1200) -> None:
|
||||
"""Block until the given host:port accepts connections or timeout expires."""
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.settimeout(1)
|
||||
if s.connect_ex((host, port)) == 0:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
raise RuntimeError(f"Server did not start on {host}:{port} within {timeout}s")
|
||||
|
||||
|
||||
def _kill_process_tree(pid: int) -> None:
|
||||
try:
|
||||
parent = psutil.Process(pid)
|
||||
children = parent.children(recursive=True)
|
||||
all_pids = [pid] + [c.pid for c in children]
|
||||
|
||||
for child in children:
|
||||
try:
|
||||
child.terminate()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
gone, alive = psutil.wait_procs(children, timeout=10)
|
||||
for child in alive:
|
||||
try:
|
||||
child.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
try:
|
||||
parent.terminate()
|
||||
parent.wait(timeout=10)
|
||||
except (psutil.NoSuchProcess, psutil.TimeoutExpired):
|
||||
try:
|
||||
parent.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
time.sleep(1)
|
||||
still_alive = [p for p in all_pids if psutil.pid_exists(p)]
|
||||
if still_alive:
|
||||
print(f"Warning: processes still alive after shutdown: {still_alive}")
|
||||
for p in still_alive:
|
||||
try:
|
||||
subprocess.run(["kill", "-9", str(p)], timeout=2)
|
||||
except Exception:
|
||||
pass
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class DiffusionServer:
|
||||
"""Start a vLLM-Omni diffusion model server as a subprocess.
|
||||
|
||||
Launched via vllm_omni.entrypoints.cli.main with the diffusion-specific
|
||||
parallelism flags (--usp, --ring, --cfg-parallel-size, etc.) passed directly
|
||||
on the CLI. Minimum hardware: 4× NVIDIA H100 80 GB.
|
||||
"""
|
||||
|
||||
server_type = "vllm-omni"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
serve_args: list[str],
|
||||
*,
|
||||
port: int | None = None,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.serve_args = serve_args
|
||||
self.host = "127.0.0.1"
|
||||
self.port = port if port is not None else _get_open_port()
|
||||
self.proc: subprocess.Popen | None = None
|
||||
self.test_name: str = ""
|
||||
|
||||
def _start_server(self) -> None:
|
||||
env = os.environ.copy()
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm_omni.entrypoints.cli.main",
|
||||
"serve",
|
||||
self.model,
|
||||
"--omni",
|
||||
"--host",
|
||||
self.host,
|
||||
"--port",
|
||||
str(self.port),
|
||||
] + self.serve_args
|
||||
|
||||
print(f"Launching DiffusionServer: {' '.join(cmd)}")
|
||||
self.proc = subprocess.Popen(
|
||||
cmd,
|
||||
env=env,
|
||||
cwd=str(Path(__file__).parent.parent.parent.parent),
|
||||
)
|
||||
_wait_for_port(self.host, self.port)
|
||||
print(f"DiffusionServer ready on {self.host}:{self.port}")
|
||||
|
||||
def __enter__(self):
|
||||
self._start_server()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
if self.proc:
|
||||
_kill_process_tree(self.proc.pid)
|
||||
|
||||
|
||||
_CACHE_DIT_REQUIRED_VERSION = os.environ.get("CACHE_DIT_VERSION", "1.3.0")
|
||||
|
||||
|
||||
def _check_cache_dit_version(required: str = _CACHE_DIT_REQUIRED_VERSION) -> None:
|
||||
"""Verify that the installed cache-dit package matches *required* exactly.
|
||||
|
||||
Raises RuntimeError if the package is not installed or the version differs.
|
||||
"""
|
||||
try:
|
||||
installed = importlib.metadata.version("cache-dit")
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
raise RuntimeError(
|
||||
f"cache-dit is not installed. Please install version {required}: pip install cache-dit=={required}"
|
||||
)
|
||||
if installed != required:
|
||||
raise RuntimeError(
|
||||
f"cache-dit version mismatch: required {required}, "
|
||||
f"but found {installed}. "
|
||||
f"Please install the correct version: pip install cache-dit=={required}"
|
||||
)
|
||||
|
||||
|
||||
class SglangServer:
|
||||
"""Start a sglang serve process for diffusion benchmarking.
|
||||
|
||||
Supports two Cache-DiT activation modes:
|
||||
1. Environment variable: pass env={"SGLANG_CACHE_DIT_ENABLED": "true"}
|
||||
2. YAML config file: pass cache_dit_config={...} (written to a temp
|
||||
file and forwarded as --cache-dit-config; requires cache-dit >= 1.3.0)
|
||||
"""
|
||||
|
||||
server_type = "sglang"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
serve_args: list[str],
|
||||
*,
|
||||
port: int | None = None,
|
||||
env_overrides: dict[str, str] | None = None,
|
||||
cache_dit_config: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.serve_args = serve_args
|
||||
self.host = "127.0.0.1"
|
||||
self.port = port if port is not None else _get_open_port()
|
||||
self.env_overrides = env_overrides or {}
|
||||
self.cache_dit_config = cache_dit_config
|
||||
self.proc: subprocess.Popen | None = None
|
||||
self._tmp_yaml: str | None = None
|
||||
self.test_name: str = ""
|
||||
if self.cache_dit_config is not None:
|
||||
_check_cache_dit_version()
|
||||
|
||||
@staticmethod
|
||||
def _write_cache_dit_yaml(config: dict[str, Any]) -> str:
|
||||
"""Serialize config dict to a temp YAML file and return its path.
|
||||
|
||||
Tries PyYAML first for clean block-style output; falls back to
|
||||
json.dump since JSON is valid YAML and correctly handles arbitrary
|
||||
nesting, lists, booleans, and null values.
|
||||
"""
|
||||
tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False)
|
||||
try:
|
||||
import yaml # PyYAML
|
||||
|
||||
yaml.dump(config, tmp, default_flow_style=False, allow_unicode=True)
|
||||
except ImportError:
|
||||
json.dump(config, tmp, indent=2, ensure_ascii=False)
|
||||
tmp.write("\n")
|
||||
tmp.close()
|
||||
print(f" Cache-DiT config written to: {tmp.name}")
|
||||
return tmp.name
|
||||
|
||||
def _start_server(self) -> None:
|
||||
env = os.environ.copy()
|
||||
env.update(self.env_overrides)
|
||||
|
||||
cmd = [
|
||||
"sglang",
|
||||
"serve",
|
||||
"--model-path",
|
||||
self.model,
|
||||
"--host",
|
||||
self.host,
|
||||
"--port",
|
||||
str(self.port),
|
||||
] + self.serve_args
|
||||
|
||||
if self.cache_dit_config is not None:
|
||||
self._tmp_yaml = self._write_cache_dit_yaml(self.cache_dit_config)
|
||||
cmd += ["--cache-dit-config", self._tmp_yaml]
|
||||
|
||||
print(f"Launching SglangServer: {' '.join(cmd)}")
|
||||
if self.env_overrides:
|
||||
print(f" Extra env: {self.env_overrides}")
|
||||
|
||||
self.proc = subprocess.Popen(
|
||||
cmd,
|
||||
env=env,
|
||||
cwd=str(Path(__file__).parent.parent.parent.parent),
|
||||
)
|
||||
_wait_for_port(self.host, self.port)
|
||||
print(f"SglangServer ready on {self.host}:{self.port}")
|
||||
|
||||
def __enter__(self):
|
||||
self._start_server()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
if self.proc:
|
||||
_kill_process_tree(self.proc.pid)
|
||||
if self._tmp_yaml:
|
||||
try:
|
||||
Path(self._tmp_yaml).unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_serve_args(serve_args_dict: dict[str, Any], server_type: str = "vllm-omni") -> list[str]:
|
||||
"""Convert a serve_args dict from test.json into a flat CLI argument list.
|
||||
|
||||
Boolean handling differs by server type:
|
||||
- vllm-omni uses store_true/store_false style: True → add flag only,
|
||||
False → omit flag entirely.
|
||||
- sglang accepts explicit boolean values: always emit ``--flag true/false``.
|
||||
"""
|
||||
args: list[str] = []
|
||||
for key, value in serve_args_dict.items():
|
||||
flag = f"--{key}"
|
||||
if isinstance(value, bool):
|
||||
if server_type == "sglang":
|
||||
args.extend([flag, str(value).lower()])
|
||||
elif value:
|
||||
args.append(flag)
|
||||
elif isinstance(value, dict):
|
||||
args.extend([flag, json.dumps(value, separators=(",", ":"))])
|
||||
else:
|
||||
args.extend([flag, str(value)])
|
||||
return args
|
||||
|
||||
|
||||
def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Return one server-config dict per unique test_name."""
|
||||
seen: set[str] = set()
|
||||
result: list[dict[str, Any]] = []
|
||||
for cfg in configs:
|
||||
test_name = cfg["test_name"]
|
||||
if test_name in seen:
|
||||
continue
|
||||
seen.add(test_name)
|
||||
server_type = cfg.get("server_type", "vllm-omni")
|
||||
result.append(
|
||||
{
|
||||
"test_name": test_name,
|
||||
"server_type": server_type,
|
||||
"model": cfg["server_params"]["model"],
|
||||
"serve_args": _build_serve_args(cfg["server_params"].get("serve_args", {}), server_type),
|
||||
"env_overrides": cfg["server_params"].get("env", {}),
|
||||
"cache_dit_config": cfg["server_params"].get("cache_dit_config"),
|
||||
"benchmark_backend": server_type, # "vllm-omni" or "sglang"
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _test_param_mapping(configs: list[dict[str, Any]]) -> dict[str, list[dict]]:
|
||||
mapping: dict[str, list[dict]] = {}
|
||||
for cfg in configs:
|
||||
name = cfg["test_name"]
|
||||
mapping.setdefault(name, [])
|
||||
mapping[name].extend(cfg["benchmark_params"])
|
||||
return mapping
|
||||
|
||||
|
||||
def _make_server(server_cfg: dict[str, Any]) -> DiffusionServer | SglangServer:
|
||||
"""Factory: return the appropriate server instance for the given config."""
|
||||
model = server_cfg["model"]
|
||||
serve_args = server_cfg["serve_args"]
|
||||
if server_cfg["server_type"] == "sglang":
|
||||
return SglangServer(
|
||||
model=model,
|
||||
serve_args=serve_args,
|
||||
env_overrides=server_cfg.get("env_overrides", {}),
|
||||
cache_dit_config=server_cfg.get("cache_dit_config"),
|
||||
)
|
||||
return DiffusionServer(model=model, serve_args=serve_args)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parametrize data
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
server_params = _unique_server_params(BENCHMARK_CONFIGS)
|
||||
test_param_map = _test_param_mapping(BENCHMARK_CONFIGS)
|
||||
|
||||
benchmark_indices: list[int] = list(range(max(len(v) for v in test_param_map.values())))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pytest fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def diffusion_server(request):
|
||||
"""Start one server (vllm-omni or sglang) per unique test configuration."""
|
||||
with _server_lock:
|
||||
server_cfg: dict[str, Any] = request.param
|
||||
test_name = server_cfg["test_name"]
|
||||
server_type = server_cfg["server_type"]
|
||||
|
||||
print(f"\nStarting {server_type} server for test: {test_name}")
|
||||
with _make_server(server_cfg) as server:
|
||||
server.test_name = test_name
|
||||
print(f"{server_type} server started successfully")
|
||||
yield server
|
||||
print(f"{server_type} server stopping…")
|
||||
|
||||
print(f"{server_type} server stopped")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def benchmark_params(request, diffusion_server):
|
||||
"""Yield the benchmark params dict for the current (server, index) pair."""
|
||||
param_index: int = request.param
|
||||
test_name = diffusion_server.test_name
|
||||
|
||||
params_list = test_param_map.get(test_name, [])
|
||||
if not params_list:
|
||||
raise ValueError(f"No benchmark params for test: {test_name}")
|
||||
if param_index >= len(params_list):
|
||||
pytest.skip(f"Param index {param_index} out of range for {test_name} (has {len(params_list)} params)")
|
||||
|
||||
current = param_index + 1
|
||||
total = len(params_list)
|
||||
print(f"\n Running benchmark {current}/{total} for {test_name}")
|
||||
return {"test_name": test_name, "params": params_list[param_index]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Benchmark runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_benchmark(
|
||||
host: str,
|
||||
port: int,
|
||||
model: str,
|
||||
params: dict[str, Any],
|
||||
test_name: str,
|
||||
backend: str = "vllm-omni",
|
||||
) -> dict[str, Any]:
|
||||
"""Run diffusion_benchmark_serving.py as a subprocess and return parsed metrics.
|
||||
|
||||
The raw metrics are written to a temporary file by the subprocess. After
|
||||
the run completes the metrics are merged with full metadata (test_name,
|
||||
backend, benchmark_params, timestamp) and appended to the session-wide
|
||||
aggregated JSON file (AGGREGATED_RESULT_FILE). The temporary file is
|
||||
removed afterwards. Subprocess stdout/stderr are tee'd to a .log file
|
||||
under BENCHMARK_RESULT_DIR/logs/; its path is stored in the record.
|
||||
"""
|
||||
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
log_dir = BENCHMARK_RESULT_DIR / "logs"
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_file = log_dir / f"{test_name}_{backend}_{timestamp}.log"
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
|
||||
tmp_result_file = Path(tmp.name)
|
||||
|
||||
exclude_keys = {"baseline", "dataset", "task", "name"}
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
BENCHMARK_SCRIPT,
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
"--model",
|
||||
model,
|
||||
"--backend",
|
||||
backend,
|
||||
"--dataset",
|
||||
params.get("dataset", "random"),
|
||||
"--task",
|
||||
params.get("task", "t2i"),
|
||||
"--output-file",
|
||||
str(tmp_result_file),
|
||||
]
|
||||
|
||||
for key, value in params.items():
|
||||
if key in exclude_keys or value is None:
|
||||
continue
|
||||
flag = f"--{key}"
|
||||
if isinstance(value, bool):
|
||||
if value:
|
||||
cmd.append(flag)
|
||||
elif isinstance(value, (dict, list)):
|
||||
cmd.extend([flag, json.dumps(value, separators=(",", ":"))])
|
||||
else:
|
||||
cmd.extend([flag, str(value)])
|
||||
|
||||
# Insert -u so the subprocess runs with unbuffered stdout/stderr, ensuring
|
||||
# all print() output is flushed to the pipe immediately instead of being
|
||||
# held in Python's internal block-buffer until process exit (which can
|
||||
# cause truncated or out-of-order log output when stdout is piped).
|
||||
cmd = [cmd[0], "-u"] + cmd[1:]
|
||||
|
||||
print(f"\nRunning benchmark (backend={backend}): {' '.join(cmd)}")
|
||||
print(f" Log file: {log_file}")
|
||||
|
||||
# Redirect stdout + stderr directly to the log file at the OS level
|
||||
# (equivalent to `cmd > log 2>&1`), so no output is ever lost regardless
|
||||
# of how the subprocess exits. The log is echoed to the terminal afterwards.
|
||||
with open(log_file, "w", encoding="utf-8") as log_fh:
|
||||
log_fh.write(f"cmd: {' '.join(cmd)}\n\n")
|
||||
log_fh.flush()
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=log_fh,
|
||||
stderr=log_fh,
|
||||
cwd=str(Path(__file__).parent.parent.parent.parent),
|
||||
)
|
||||
process.wait()
|
||||
|
||||
with open(log_file, encoding="utf-8") as log_fh:
|
||||
print(log_fh.read(), end="")
|
||||
|
||||
if process.returncode != 0:
|
||||
tmp_result_file.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"Benchmark script exited with code {process.returncode}")
|
||||
|
||||
if not tmp_result_file.exists():
|
||||
raise FileNotFoundError(f"Benchmark result file not found: {tmp_result_file}")
|
||||
|
||||
try:
|
||||
with open(tmp_result_file, encoding="utf-8") as f:
|
||||
metrics: dict[str, Any] = json.load(f)
|
||||
finally:
|
||||
tmp_result_file.unlink(missing_ok=True)
|
||||
|
||||
record: dict[str, Any] = {
|
||||
"test_name": test_name,
|
||||
"backend": backend,
|
||||
"timestamp": timestamp,
|
||||
"benchmark_params": params,
|
||||
"result": metrics,
|
||||
"log_file": str(log_file),
|
||||
}
|
||||
_append_to_aggregated_file(record)
|
||||
print(f"\n Result appended to: {AGGREGATED_RESULT_FILE}")
|
||||
print(f" Log saved to: {log_file}")
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
|
||||
"""Assert that benchmark metrics satisfy the configured baselines."""
|
||||
num_prompts = params.get("num-prompts", 10)
|
||||
completed = result.get("completed_requests", result.get("completed", 0))
|
||||
assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
|
||||
|
||||
for metric, threshold in params.get("baseline", {}).items():
|
||||
current = result.get(metric)
|
||||
assert current is not None, f"Metric '{metric}' not found in result: {list(result.keys())}"
|
||||
if "throughput" in metric:
|
||||
assert current >= threshold, f"{metric}: {current:.4f} < baseline {threshold}"
|
||||
else:
|
||||
assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"diffusion_server",
|
||||
server_params,
|
||||
ids=[p["test_name"] for p in server_params],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
|
||||
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
|
||||
"""Run the diffusion performance benchmark and assert against baselines.
|
||||
|
||||
One server is started per unique parallel configuration (module scope).
|
||||
For each server, all benchmark parameter sets defined in the config JSON
|
||||
are executed sequentially; results are asserted against the baselines.
|
||||
|
||||
Tracked metrics:
|
||||
- throughput_qps (higher is better)
|
||||
- latency_p50, latency_p99 (lower is better)
|
||||
"""
|
||||
test_name = benchmark_params["test_name"]
|
||||
params = benchmark_params["params"]
|
||||
backend = diffusion_server.server_type # "vllm-omni" or "sglang"
|
||||
|
||||
result = run_benchmark(
|
||||
host=diffusion_server.host,
|
||||
port=diffusion_server.port,
|
||||
model=diffusion_server.model,
|
||||
params=params,
|
||||
test_name=test_name,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Results for {test_name} (server={diffusion_server.server_type}, backend={backend}):")
|
||||
for key in (
|
||||
"throughput_qps",
|
||||
"latency_mean",
|
||||
"latency_median",
|
||||
"latency_p50",
|
||||
"latency_p99",
|
||||
"peak_memory_mb_max",
|
||||
"peak_memory_mb_mean",
|
||||
"peak_memory_mb_median",
|
||||
):
|
||||
if key in result:
|
||||
print(f" {key}: {result[key]:.4f}")
|
||||
|
||||
print(f"\n Aggregated results: {AGGREGATED_RESULT_FILE}")
|
||||
print("=" * 60)
|
||||
|
||||
assert_result(result, params)
|
||||
@@ -0,0 +1,57 @@
|
||||
[
|
||||
{
|
||||
"name": "512x512_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.001,
|
||||
"latency_p99": 1000.0,
|
||||
"peak_memory_mb_max": 400000,
|
||||
"peak_memory_mb_mean": 400000
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1536,
|
||||
"height": 1536,
|
||||
"num-inference-steps": 35,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.001,
|
||||
"latency_p99": 1000.0,
|
||||
"peak_memory_mb_max": 400000,
|
||||
"peak_memory_mb_mean": 400000
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "mixed_resolution",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{"width": 512, "height": 512, "num_inference_steps": 20, "weight": 0.15},
|
||||
{"width": 768, "height": 768, "num_inference_steps": 20, "weight": 0.25},
|
||||
{"width": 1024, "height": 1024, "num_inference_steps": 25, "weight": 0.45},
|
||||
{"width": 1536, "height": 1536, "num_inference_steps": 35, "weight": 0.15}
|
||||
],
|
||||
"baseline": {
|
||||
"throughput_qps": 0.001,
|
||||
"latency_p99": 1000.0,
|
||||
"peak_memory_mb_max": 400000,
|
||||
"peak_memory_mb_mean": 400000
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,73 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_sglang_qwen_image_single_device",
|
||||
"description": "sglang: single-device baseline (no parallelism)",
|
||||
"server_type": "sglang",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"num-gpus": 1,
|
||||
"dit-cpu-offload": false,
|
||||
"text-encoder-cpu-offload": false,
|
||||
"vae-cpu-offload": false
|
||||
}
|
||||
},
|
||||
"benchmark_params": {"$ref": "benchmark_params.json"}
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_sglang_qwen_image_ulysses2_cfg2_vae_patch4",
|
||||
"description": "sglang: USP=2 + CFG-parallel=2 + VAE patch parallel via --use-parallel-tiling",
|
||||
"server_type": "sglang",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"num-gpus": 4,
|
||||
"sp-degree": 2,
|
||||
"ulysses-degree": 2,
|
||||
"enable-cfg-parallel": true,
|
||||
"use-parallel-tiling": true,
|
||||
"dit-cpu-offload": false,
|
||||
"text-encoder-cpu-offload": false,
|
||||
"vae-cpu-offload": false
|
||||
}
|
||||
},
|
||||
"benchmark_params": {"$ref": "benchmark_params.json"}
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_sglang_qwen_image_ulysses2_cfg2_cache_dit",
|
||||
"description": "sglang: USP=2 + CFG-parallel=2 + Cache-DiT via YAML config (--cache-dit-config)",
|
||||
"server_type": "sglang",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"cache_dit_config": {
|
||||
"cache_config": {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": false,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": "none",
|
||||
"scm_steps_policy": "dynamic"
|
||||
},
|
||||
"parallelism_config": {
|
||||
"ulysses_size": 2,
|
||||
"attention_backend": "native"
|
||||
}
|
||||
},
|
||||
"serve_args": {
|
||||
"num-gpus": 4,
|
||||
"sp-degree": 2,
|
||||
"ulysses-degree": 2,
|
||||
"enable-cfg-parallel": true,
|
||||
"dit-cpu-offload": false,
|
||||
"text-encoder-cpu-offload": false,
|
||||
"vae-cpu-offload": false
|
||||
}
|
||||
},
|
||||
"benchmark_params": {"$ref": "benchmark_params.json"}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,58 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_qwen_image_single_device",
|
||||
"description": "Single-device baseline (no parallelism)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": {"$ref": "benchmark_params.json"}
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4",
|
||||
"description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"vae-patch-parallel-size": 4,
|
||||
"vae-use-tiling": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": {"$ref": "benchmark_params.json"}
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_qwen_image_ulysses2_cfg2_cache_dit",
|
||||
"description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"serve_args": {
|
||||
"ulysses-degree": 2,
|
||||
"cfg-parallel-size": 2,
|
||||
"cache-backend": "cache_dit",
|
||||
"cache-config": {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": false,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": null,
|
||||
"scm_steps_policy": "dynamic"
|
||||
},
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": {"$ref": "benchmark_params.json"}
|
||||
}
|
||||
]
|
||||
@@ -651,6 +651,9 @@ class DiffusionOutput:
|
||||
# logged duration of stages
|
||||
stage_durations: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
# memory usage info
|
||||
peak_memory_mb: float = 0.0
|
||||
|
||||
|
||||
class AttentionBackendEnum(enum.Enum):
|
||||
FA = enum.auto()
|
||||
|
||||
@@ -180,6 +180,7 @@ class DiffusionEngine:
|
||||
multimodal_output={"audio": request_audio_payload},
|
||||
final_output_type="audio",
|
||||
stage_durations=output.stage_durations,
|
||||
peak_memory_mb=output.peak_memory_mb,
|
||||
),
|
||||
]
|
||||
else:
|
||||
@@ -196,6 +197,7 @@ class DiffusionEngine:
|
||||
custom_output=output.custom_output or {},
|
||||
multimodal_output=mm_output,
|
||||
stage_durations=output.stage_durations,
|
||||
peak_memory_mb=output.peak_memory_mb,
|
||||
),
|
||||
]
|
||||
else:
|
||||
@@ -226,6 +228,7 @@ class DiffusionEngine:
|
||||
multimodal_output={"audio": request_audio_payload},
|
||||
final_output_type="audio",
|
||||
stage_durations=output.stage_durations,
|
||||
peak_memory_mb=output.peak_memory_mb,
|
||||
),
|
||||
)
|
||||
else:
|
||||
@@ -252,6 +255,7 @@ class DiffusionEngine:
|
||||
custom_output=output.custom_output or {},
|
||||
multimodal_output=mm_output,
|
||||
stage_durations=output.stage_durations,
|
||||
peak_memory_mb=output.peak_memory_mb,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -190,6 +190,33 @@ class DiffusionModelRunner:
|
||||
"""Load weights into the pipeline."""
|
||||
return self.pipeline.load_weights(weights)
|
||||
|
||||
def _record_peak_memory(self, output: DiffusionOutput) -> None:
|
||||
"""Record peak GPU memory for the current forward pass into output.
|
||||
|
||||
Must be called immediately after pipeline.forward(), with
|
||||
reset_peak_memory_stats() called just before it, so the measurement
|
||||
reflects this request only and not the global historical maximum.
|
||||
|
||||
Uses max_memory_reserved (CUDA memory pool high-water mark) rather than
|
||||
max_memory_allocated so that allocator fragmentation is also visible.
|
||||
See: https://docs.pytorch.org/docs/stable/generated/torch.cuda.memory.max_memory_reserved.html
|
||||
"""
|
||||
peak_reserved_bytes = current_omni_platform.max_memory_reserved()
|
||||
peak_allocated_bytes = current_omni_platform.max_memory_allocated()
|
||||
|
||||
output.peak_memory_mb = peak_reserved_bytes / (1024**2)
|
||||
peak_reserved_gb = peak_reserved_bytes / (1024**3)
|
||||
peak_allocated_gb = peak_allocated_bytes / (1024**3)
|
||||
pool_overhead_gb = peak_reserved_gb - peak_allocated_gb
|
||||
|
||||
logger.info(
|
||||
"Peak GPU memory (this request): %.2f GB reserved, %.2f GB allocated, %.2f GB pool overhead (%.1f%%)",
|
||||
peak_reserved_gb,
|
||||
peak_allocated_gb,
|
||||
pool_overhead_gb,
|
||||
pool_overhead_gb / peak_reserved_gb * 100 if peak_reserved_gb > 0 else 0.0,
|
||||
)
|
||||
|
||||
def execute_model(self, req: OmniDiffusionRequest) -> DiffusionOutput:
|
||||
"""
|
||||
Execute a forward pass for the given requests.
|
||||
@@ -239,10 +266,17 @@ class DiffusionModelRunner:
|
||||
):
|
||||
self.cache_backend.refresh(self.pipeline, req.sampling_params.num_inference_steps)
|
||||
|
||||
is_primary = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
|
||||
if is_primary:
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
|
||||
with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config):
|
||||
with record_function("pipeline_forward"):
|
||||
output = self.pipeline.forward(req)
|
||||
|
||||
if is_primary:
|
||||
self._record_peak_memory(output)
|
||||
|
||||
# NOTE:
|
||||
if (
|
||||
self.cache_backend is not None
|
||||
|
||||
@@ -246,6 +246,7 @@ class OmniBase:
|
||||
req_id = result.get("request_id")
|
||||
engine_outputs = result.get("engine_outputs")
|
||||
stage_durations = getattr(result["engine_outputs"], "stage_durations", {})
|
||||
peak_memory_mb = getattr(result["engine_outputs"], "peak_memory_mb", 0.0)
|
||||
finished = engine_outputs.finished
|
||||
|
||||
submit_ts = result.get("stage_submit_ts")
|
||||
@@ -281,6 +282,7 @@ class OmniBase:
|
||||
request_output=engine_outputs,
|
||||
images=images,
|
||||
stage_durations=stage_durations,
|
||||
peak_memory_mb=peak_memory_mb,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
|
||||
@@ -1916,6 +1916,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
|
||||
# Handle profiling data
|
||||
stage_durations = omni_outputs.stage_durations
|
||||
peak_memory_mb = omni_outputs.peak_memory_mb
|
||||
|
||||
# Handle different image output formats
|
||||
images = []
|
||||
@@ -1971,6 +1972,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
"url": f"data:image/png;base64,{img_base64}",
|
||||
},
|
||||
"stage_durations": stage_durations,
|
||||
"peak_memory_mb": peak_memory_mb,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -2187,6 +2189,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
# Handle nested OmniRequestOutput structure where images might be in request_output
|
||||
images = getattr(result.request_output, "images", [])
|
||||
stage_durations = result.stage_durations
|
||||
peak_memory_mb = result.peak_memory_mb
|
||||
|
||||
# Convert images to base64 content
|
||||
image_contents: list[dict[str, Any]] = []
|
||||
@@ -2208,6 +2211,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
"url": f"data:image/png;base64,{img_base64}",
|
||||
},
|
||||
"stage_durations": stage_durations,
|
||||
"peak_memory_mb": peak_memory_mb,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -65,6 +65,9 @@ class OmniRequestOutput:
|
||||
# profiling data
|
||||
stage_durations: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
# memory usage info
|
||||
peak_memory_mb: float = 0.0
|
||||
|
||||
@classmethod
|
||||
def from_pipeline(
|
||||
cls,
|
||||
@@ -102,6 +105,7 @@ class OmniRequestOutput:
|
||||
custom_output: dict[str, Any] | None = None,
|
||||
final_output_type: str = "image",
|
||||
stage_durations: dict[str, float] | None = None,
|
||||
peak_memory_mb: float = 0.0,
|
||||
) -> "OmniRequestOutput":
|
||||
"""Create output from diffusion model.
|
||||
|
||||
@@ -113,6 +117,8 @@ class OmniRequestOutput:
|
||||
latents: Optional latent tensors
|
||||
multimodal_output: Optional multimodal output dict
|
||||
custom_output: Optional custom output dict (e.g. latent trajectories, prompt embeds)
|
||||
stage_durations: Optional stage durations (execution time of each stage) dict
|
||||
peak_memory_mb: Peak memory usage in MB
|
||||
|
||||
Returns:
|
||||
OmniRequestOutput configured for diffusion mode
|
||||
@@ -127,6 +133,7 @@ class OmniRequestOutput:
|
||||
_multimodal_output=multimodal_output or {},
|
||||
_custom_output=custom_output or {},
|
||||
stage_durations=stage_durations or {},
|
||||
peak_memory_mb=peak_memory_mb,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
@@ -277,6 +284,7 @@ class OmniRequestOutput:
|
||||
f"multimodal_output={self._multimodal_output}",
|
||||
f"custom_output={self._custom_output}",
|
||||
f"stage_durations={self.stage_durations}",
|
||||
f"peak_memory_mb={self.peak_memory_mb}",
|
||||
]
|
||||
|
||||
return f"OmniRequestOutput({', '.join(parts)})"
|
||||
Reference in new issue
Block a user